Delta Storage: initial project setup

The first PR #925 was accidentally merged into master and then reverted.

So, here's the same PR again.

Closes #930

Signed-off-by: Scott Sandre <scott.sandre@databricks.com>
GitOrigin-RevId: 22f053c68533b8e638ef835d436b974cc4086f80

build.sbt

@@ -33,6 +33,7 @@ lazy val commonSettings = Seq(
 )
 
 lazy val core = (project in file("core"))
+  .dependsOn(storage)
   .enablePlugins(GenJavadocPlugin, JavaUnidocPlugin, ScalaUnidocPlugin, Antlr4Plugin)
   .settings (
     name := "delta-core",
@@ -158,6 +159,19 @@ lazy val contribs = (project in file("contribs"))
     Compile / compile := ((Compile / compile) dependsOn createTargetClassesDir).value
   )
 
+// TODO javastyle tests
+// TODO unidoc
+lazy val storage = (project in file("storage"))
+  .settings (
+    name := "delta-storage",
+    commonSettings,
+    releaseSettings,
+    libraryDependencies ++= Seq(
+      // User can provide any 2.x or 3.x version. We don't use any new fancy APIs.
+      "org.apache.hadoop" % "hadoop-client-api" % "3.1.0" % "provided"
+    )
+  )
+
 /**
  * Get list of python files and return the mapping between source files and target paths
  * in the generated package JAR.

project/MimaExcludes.scala

@@ -22,6 +22,7 @@ import com.typesafe.tools.mima.core.ProblemFilters._
  */
 object MimaExcludes {
   val ignoredABIProblems = Seq(
+      // scalastyle:off line.size.limit
       ProblemFilters.exclude[Problem]("org.*"),
       ProblemFilters.exclude[Problem]("io.delta.sql.parser.*"),
       ProblemFilters.exclude[Problem]("io.delta.tables.execution.*"),
@@ -77,7 +78,13 @@ object MimaExcludes {
       ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaMergeBuilder.initializeLogIfNecessary$default$2"),
 
       // Changes in 0.7.0
-      ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaTable.makeUpdateTable")
+      ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaTable.makeUpdateTable"),
+
+      // Changes in 1.2.0
+      ProblemFilters.exclude[MissingClassProblem]("io.delta.storage.LogStore"),
+      ProblemFilters.exclude[MissingClassProblem]("io.delta.storage.CloseableIterator")
+
+      // scalastyle:on line.size.limit
   )
 }
 

core/src/main/scala/io/delta/storage/CloseableIterator.java → storage/src/main/scala/io/delta/storage/CloseableIterator.java

@@ -16,8 +16,6 @@
 
 package io.delta.storage;
 
-import org.apache.spark.annotation.DeveloperApi;
-
 import java.io.Closeable;
 import java.util.Iterator;
 
@@ -29,5 +27,4 @@
  *
  * @since 1.0.0
  */
-@DeveloperApi
 public interface CloseableIterator<T> extends Iterator<T>, Closeable {}

core/src/main/scala/io/delta/storage/LogStore.java → storage/src/main/scala/io/delta/storage/LogStore.java

@@ -19,7 +19,6 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.Path;
-import org.apache.spark.annotation.DeveloperApi;
 
 import java.io.FileNotFoundException;
 import java.nio.file.FileAlreadyExistsException;
@@ -50,7 +49,6 @@
  *
  * @since 1.0.0
  */
-@DeveloperApi
 public abstract class LogStore {
 
   private Configuration initHadoopConf;