Reformat files with scalafmt (#473)

dev/.scalafmt.conf

@@ -1,3 +1,5 @@
+version = 2.6.3
+
 # The following configs are taken from https://github.com/apache/spark/blob/master/dev/.scalafmt.conf
 align = none
 align.openParenDefnSite = false

src/main/scala-spark2/com/microsoft/hyperspace/index/sources/iceberg/IcebergShims.scala

@@ -32,7 +32,8 @@ import com.microsoft.hyperspace.util.JavaConverters._
 
 object IcebergShims {
 
-  def isIcebergRelation(plan: LogicalPlan): Boolean = plan match {
+  def isIcebergRelation(plan: LogicalPlan): Boolean =
+    plan match {
       case DataSourceV2Relation(_: IcebergSource, _, _, _, _) => true
       case _ => false
     }

src/main/scala-spark3.0/com/microsoft/hyperspace/index/sources/iceberg/IcebergShims.scala

@@ -28,7 +28,8 @@ object IcebergShims {
 
   // In Spark 3, the V2ScanRelationPushdown rule can convert DataSourceV2Relation into
   // DataSourceV2ScanRelation.
-  def isIcebergRelation(plan: LogicalPlan): Boolean = plan match {
+  def isIcebergRelation(plan: LogicalPlan): Boolean =
+    plan match {
       case DataSourceV2Relation(_: SparkTable, _, _, _, _) => true
       case DataSourceV2ScanRelation(_: SparkTable, _, _) => true
       case _ => false

src/main/scala-spark3.1/com/microsoft/hyperspace/index/sources/iceberg/IcebergShims.scala

@@ -26,7 +26,8 @@ import org.apache.spark.util.hyperspace.Utils
 
 object IcebergShims {
 
-  def isIcebergRelation(plan: LogicalPlan): Boolean = plan match {
+  def isIcebergRelation(plan: LogicalPlan): Boolean =
+    plan match {
       case DataSourceV2Relation(_: SparkTable, _, _, _, _) => true
       case DataSourceV2ScanRelation(DataSourceV2Relation(_: SparkTable, _, _, _, _), _, _) => true
       case _ => false

src/main/scala/com/fasterxml/jackson/hyperspace/module/scala/ScalaObjectMapper.scala

@@ -78,25 +78,29 @@ trait ScalaObjectMapper {
     if (isArray(clazz)) {
       val typeArguments = m.typeArguments.map(constructType(_)).toArray
       if (typeArguments.length != 1) {
-        throw new IllegalArgumentException("Need exactly 1 type parameter for array like types ("+clazz.getName+")")
+        throw new IllegalArgumentException(
+          "Need exactly 1 type parameter for array like types (" + clazz.getName + ")")
       }
       getTypeFactory.constructArrayType(typeArguments(0))
     } else if (isMapLike(clazz)) {
       val typeArguments = m.typeArguments.map(constructType(_)).toArray
       if (typeArguments.length != 2) {
-        throw new IllegalArgumentException("Need exactly 2 type parameters for map like types ("+clazz.getName+")")
+        throw new IllegalArgumentException(
+          "Need exactly 2 type parameters for map like types (" + clazz.getName + ")")
       }
       getTypeFactory.constructMapLikeType(clazz, typeArguments(0), typeArguments(1))
     } else if (isReference(clazz)) { // Option is a subclss of IterableOnce, so check it first
       val typeArguments = m.typeArguments.map(constructType(_)).toArray
       if (typeArguments.length != 1) {
-        throw new IllegalArgumentException("Need exactly 1 type parameter for reference types ("+clazz.getName+")")
+        throw new IllegalArgumentException(
+          "Need exactly 1 type parameter for reference types (" + clazz.getName + ")")
       }
       getTypeFactory.constructReferenceType(clazz, typeArguments(0))
     } else if (isCollectionLike(clazz)) {
       val typeArguments = m.typeArguments.map(constructType(_)).toArray
       if (typeArguments.length != 1) {
-        throw new IllegalArgumentException("Need exactly 1 type parameter for collection like types ("+clazz.getName+")")
+        throw new IllegalArgumentException(
+          "Need exactly 1 type parameter for collection like types (" + clazz.getName + ")")
       }
       getTypeFactory.constructCollectionLikeType(clazz, typeArguments(0))
     } else {

src/main/scala/com/microsoft/hyperspace/Hyperspace.scala

@@ -156,8 +156,8 @@ class Hyperspace(spark: SparkSession) {
    * @param redirectFunc optional function to redirect output of explain.
    * @param verbose Flag to enable verbose mode.
    */
-  def explain(df: DataFrame, verbose: Boolean = false)(
+  def explain(df: DataFrame, verbose: Boolean = false)(implicit
-      implicit redirectFunc: String => Unit = print): Unit = {
+      redirectFunc: String => Unit = print): Unit = {
     redirectFunc(PlanAnalyzer.explainString(df, spark, indexManager.indexes, verbose))
   }
 

src/main/scala/com/microsoft/hyperspace/actions/CancelAction.scala

@@ -61,12 +61,7 @@ class CancelAction(final override protected val logManager: IndexLogManager) ext
    */
   final override def op(): Unit = {}
 
-  final override protected def event(
+  final override protected def event(appInfo: AppInfo, message: String): HyperspaceEvent = {
-      appInfo: AppInfo,
+    CancelActionEvent(appInfo, logEntry.asInstanceOf[IndexLogEntry], message)
-      message: String): HyperspaceEvent = {
-    CancelActionEvent(
-      appInfo,
-      logEntry.asInstanceOf[IndexLogEntry],
-      message)
   }
 }

src/main/scala/com/microsoft/hyperspace/actions/CreateAction.scala

@@ -66,8 +66,7 @@ class CreateAction(
     }
 
     // TODO: Temporarily block creating indexes using nested columns until it's fully supported.
-    if (!(HyperspaceConf.nestedColumnEnabled(spark) || resolvedColumns.get.forall(
+    if (!(HyperspaceConf.nestedColumnEnabled(spark) || resolvedColumns.get.forall(!_.isNested))) {
-          !_.isNested))) {
       throw HyperspaceException("Hyperspace does not support nested columns yet.")
     }
 

src/main/scala/com/microsoft/hyperspace/actions/RefreshIncrementalAction.scala

@@ -72,12 +72,8 @@ class RefreshIncrementalAction(
     } else {
       None
     }
-    updatedIndex = Some(
+    updatedIndex = Some(previousIndexLogEntry.derivedDataset
-      previousIndexLogEntry.derivedDataset.refreshIncremental(
+      .refreshIncremental(this, appendedSourceData, deletedFiles, previousIndexLogEntry.content))
-        this,
-        appendedSourceData,
-        deletedFiles,
-        previousIndexLogEntry.content))
   }
 
   /**

src/main/scala/com/microsoft/hyperspace/index/CoveringIndex.scala

@@ -146,7 +146,8 @@ case class CoveringIndex(
       indexData)
   }
 
-  override def equals(o: Any): Boolean = o match {
+  override def equals(o: Any): Boolean =
+    o match {
       case that: CoveringIndex => comparedData == that.comparedData
       case _ => false
     }

src/main/scala/com/microsoft/hyperspace/index/Index.scala

@@ -28,11 +28,7 @@ import org.apache.spark.sql.DataFrame
  *
  * The framework manages various types of indexes through this interface.
  */
-@JsonTypeInfo(
+@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "type")
-  use = JsonTypeInfo.Id.CLASS,
-  include = JsonTypeInfo.As.PROPERTY,
-  property = "type"
-)
 trait Index {
 
   /**

src/main/scala/com/microsoft/hyperspace/index/IndexLogEntry.scala

@@ -50,7 +50,9 @@ case class Content(root: Directory, fingerprint: NoOpFingerprint = NoOpFingerpri
     rec(
       new Path(root.name),
       root,
-      (f, prefix) =>
+      (
+          f,
+          prefix) =>
         FileInfo(new Path(prefix, f.name).toString, f.size, f.modifiedTime, f.id)).toSet
   }
 
@@ -101,9 +103,7 @@ object Content {
    * @param fileIdTracker FileIdTracker to keep mapping of file properties to assigned file ids.
    * @return Content object with Directory tree from leaf files.
    */
-  def fromLeafFiles(
+  def fromLeafFiles(files: Seq[FileStatus], fileIdTracker: FileIdTracker): Option[Content] = {
-      files: Seq[FileStatus],
-      fileIdTracker: FileIdTracker): Option[Content] = {
     if (files.nonEmpty) {
       Some(Content(Directory.fromLeafFiles(files, fileIdTracker)))
     } else {
@@ -227,9 +227,7 @@ object Directory {
    *                      updated by adding it to the files it is tracking.
    * @return Content object with Directory tree from leaf files.
    */
-  def fromLeafFiles(
+  def fromLeafFiles(files: Seq[FileStatus], fileIdTracker: FileIdTracker): Directory = {
-      files: Seq[FileStatus],
-      fileIdTracker: FileIdTracker): Directory = {
     require(
       files.nonEmpty,
       s"Empty files list found while creating a ${Directory.getClass.getName}.")
@@ -308,7 +306,8 @@ object Directory {
 // id is a unique identifier generated by Hyperspace, for each unique combination of
 // file's name, size and modifiedTime.
 case class FileInfo(name: String, size: Long, modifiedTime: Long, id: Long) {
-  override def equals(o: Any): Boolean = o match {
+  override def equals(o: Any): Boolean =
+    o match {
       case that: FileInfo =>
         name.equals(that.name) &&
           size.equals(that.size) &&
@@ -349,9 +348,7 @@ object LogicalPlanFingerprint {
  * @param appendedFiles Appended files.
  * @param deletedFiles Deleted files.
  */
-case class Update(
+case class Update(appendedFiles: Option[Content] = None, deletedFiles: Option[Content] = None)
-    appendedFiles: Option[Content] = None,
-    deletedFiles: Option[Content] = None)
 
 // IndexLogEntry-specific Hdfs that represents the source data.
 case class Hdfs(properties: Hdfs.Properties) {
@@ -467,24 +464,18 @@ case class IndexLogEntry(
     def toFileStatus(f: FileInfo) = {
       new FileStatus(f.size, false, 0, 1, f.modifiedTime, new Path(f.name))
     }
-    copy(
+    copy(source = source.copy(plan = source.plan.copy(properties = source.plan.properties.copy(
-      source = source.copy(
-        plan = source.plan.copy(
-          properties = source.plan.properties.copy(
       fingerprint = latestFingerprint,
       relations = Seq(
-              relations.head.copy(
+        relations.head.copy(data = relations.head.data.copy(properties =
-                data = relations.head.data.copy(
+          relations.head.data.properties.copy(update = Some(Update(
-                  properties = relations.head.data.properties.copy(
+            appendedFiles = Content.fromLeafFiles(appended.map(toFileStatus), fileIdTracker),
-                    update = Some(
-                      Update(
-                        appendedFiles =
-                          Content.fromLeafFiles(appended.map(toFileStatus), fileIdTracker),
             deletedFiles =
               Content.fromLeafFiles(deleted.map(toFileStatus), fileIdTracker)))))))))))
   }
 
-  override def equals(o: Any): Boolean = o match {
+  override def equals(o: Any): Boolean =
+    o match {
       case that: IndexLogEntry =>
         name.equals(that.name) &&
           derivedDataset.equals(that.derivedDataset) &&
@@ -506,8 +497,9 @@ case class IndexLogEntry(
 
   def hasParquetAsSourceFormat: Boolean = {
     relations.head.fileFormat.equals("parquet") ||
-      derivedDataset.properties.getOrElse(
+    derivedDataset.properties
-        IndexConstants.HAS_PARQUET_AS_SOURCE_FORMAT_PROPERTY, "false").toBoolean
+      .getOrElse(IndexConstants.HAS_PARQUET_AS_SOURCE_FORMAT_PROPERTY, "false")
+      .toBoolean
   }
 
   @JsonIgnore
@@ -594,8 +586,7 @@ object IndexLogEntry {
       derivedDataset,
       content,
       source,
-      properties + ((IndexConstants.HYPERSPACE_VERSION_PROPERTY, BuildInfo.version))
+      properties + ((IndexConstants.HYPERSPACE_VERSION_PROPERTY, BuildInfo.version)))
-    )
   }
 }
 
@@ -636,8 +627,7 @@ class FileIdTracker {
     setSizeHint(files.size)
     files.foreach { f =>
       if (f.id == IndexConstants.UNKNOWN_FILE_ID) {
-        throw HyperspaceException(
+        throw HyperspaceException(s"Cannot add file info with unknown id. (file: ${f.name}).")
-          s"Cannot add file info with unknown id. (file: ${f.name}).")
       }
 
       val key = (f.name, f.size, f.modifiedTime)
@@ -665,8 +655,7 @@ class FileIdTracker {
    */
   def addFile(file: FileStatus): Long = {
     fileToIdMap.getOrElseUpdate(
-      (file.getPath.toString, file.getLen, file.getModificationTime),
+      (file.getPath.toString, file.getLen, file.getModificationTime), {
-      {
         maxId += 1
         maxId
       })

src/main/scala/com/microsoft/hyperspace/index/IndexLogManager.scala

@@ -102,7 +102,8 @@ class IndexLogManagerImpl(indexPath: Path, hadoopConfiguration: Configuration =
           if (entry.exists(e => Constants.STABLE_STATES.contains(e.state))) {
             return entry
           }
-          if (entry.exists(e => e.state.equals(Constants.States.CREATING)
+          if (entry.exists(e =>
+              e.state.equals(Constants.States.CREATING)
                 || e.state.equals(Constants.States.VACUUMING))) {
             // Do not consider unrelated logs before creating or vacuuming state.
             return None

src/main/scala/com/microsoft/hyperspace/index/IndexStatistics.scala

@@ -55,12 +55,8 @@ private[hyperspace] case class IndexStatistics(
     additionalStats: Map[String, String])
 
 private[hyperspace] object IndexStatistics {
-  val INDEX_SUMMARY_COLUMNS: Seq[String] = Seq(
+  val INDEX_SUMMARY_COLUMNS: Seq[String] =
-    "name",
+    Seq("name", "indexedColumns", "indexLocation", "state", "additionalStats")
-    "indexedColumns",
-    "indexLocation",
-    "state",
-    "additionalStats")
 
   /**
    * Create IndexStatistics instance for a given IndexLogEntry.

src/main/scala/com/microsoft/hyperspace/index/execution/BucketUnionStrategy.scala

@@ -26,7 +26,8 @@ import com.microsoft.hyperspace.index.plans.logical.BucketUnion
  * to [[BucketUnionExec]] (Spark Plan)
  */
 private[hyperspace] object BucketUnionStrategy extends SparkStrategy {
-  override def apply(plan: LogicalPlan): Seq[SparkPlan] = plan match {
+  override def apply(plan: LogicalPlan): Seq[SparkPlan] =
+    plan match {
       case p: BucketUnion =>
         BucketUnionExec(p.children.map(planLater), p.bucketSpec) :: Nil
       case _ => Nil

src/main/scala/com/microsoft/hyperspace/index/rules/ApplyHyperspace.scala

@@ -32,8 +32,8 @@ import com.microsoft.hyperspace.telemetry.HyperspaceEventLogging
  * Collect candidate indexes for each source plan.
  */
 object CandidateIndexCollector extends ActiveSparkSession {
-  private val sourceFilters
+  private val sourceFilters: Seq[SourcePlanIndexFilter] =
-    : Seq[SourcePlanIndexFilter] = ColumnSchemaFilter :: FileSignatureFilter :: Nil
+    ColumnSchemaFilter :: FileSignatureFilter :: Nil
 
   private def initializePlanToIndexes(
       plan: LogicalPlan,
@@ -56,7 +56,9 @@ object CandidateIndexCollector extends ActiveSparkSession {
     val planToIndexes = initializePlanToIndexes(plan, allIndexes)
     planToIndexes.flatMap {
       case (node, allIndexes) =>
-        Some(node, sourceFilters.foldLeft(allIndexes) { (indexes, filter) =>
+        Some(
+          node,
+          sourceFilters.foldLeft(allIndexes) { (indexes, filter) =>
             filter(node, indexes)
           }).filter(_._2.nonEmpty)
     }

src/main/scala/com/microsoft/hyperspace/index/rules/JoinIndexRule.scala

@@ -52,12 +52,14 @@ object JoinPlanNodeFilter extends QueryPlanIndexFilter {
         val left = RuleUtils.getRelation(spark, l)
         val right = RuleUtils.getRelation(spark, r)
 
-        if (!(left.isDefined && right.isDefined && !RuleUtils.isIndexApplied(left.get) && !RuleUtils
+        if (!(left.isDefined && right.isDefined && !RuleUtils.isIndexApplied(
+            left.get) && !RuleUtils
             .isIndexApplied(right.get))) {
           return Map.empty
         }
 
-        val leftAndRightIndexes = candidateIndexes.getOrElse(left.get.plan, Nil) ++ candidateIndexes
+        val leftAndRightIndexes =
+          candidateIndexes.getOrElse(left.get.plan, Nil) ++ candidateIndexes
             .getOrElse(right.get.plan, Nil)
 
         val joinConditionCond = withFilterReasonTag(
@@ -206,7 +208,6 @@ object JoinAttributeFilter extends QueryPlanIndexFilter {
    * E.g. (A = B and A = D) is not supported. A maps with both B and D. There isn't a one-to-one
    * mapping.
    *
-   *
    * Background knowledge:
    * An alias in a query plan is represented as [[Alias]] at the time of
    * its creation. Unnecessary aliases get resolved and removed during query analysis phase by

src/main/scala/com/microsoft/hyperspace/index/rules/RuleUtils.scala

@@ -388,7 +388,8 @@ object RuleUtils {
     // Extract top level plan including all required columns for shuffle in its output.
     object ExtractTopLevelPlanForShuffle {
       type returnType = (LogicalPlan, Seq[Option[Attribute]], Boolean)
-      def unapply(plan: LogicalPlan): Option[returnType] = plan match {
+      def unapply(plan: LogicalPlan): Option[returnType] =
+        plan match {
           case p @ Project(_, Filter(_, LogicalRelation(_: HadoopFsRelation, _, _, _))) =>
             Some(p, getIndexedAttrs(p, bucketSpec.bucketColumnNames), true)
           case p @ Project(_, LogicalRelation(_: HadoopFsRelation, _, _, _)) =>

src/main/scala/com/microsoft/hyperspace/index/sources/FileBasedSourceProviderManager.scala

@@ -37,9 +37,11 @@ import com.microsoft.hyperspace.util.{CacheWithTransform, HyperspaceConf}
  */
 class FileBasedSourceProviderManager(spark: SparkSession) {
   private val sourceProviders: CacheWithTransform[String, Seq[FileBasedSourceProvider]] =
-    new CacheWithTransform[String, Seq[FileBasedSourceProvider]]({ () =>
+    new CacheWithTransform[String, Seq[FileBasedSourceProvider]](
+      { () =>
         HyperspaceConf.fileBasedSourceBuilders(spark)
-    }, { builderClassNames =>
+      },
+      { builderClassNames =>
         buildProviders(builderClassNames)
       })
 

src/main/scala/com/microsoft/hyperspace/index/sources/default/DefaultFileBasedRelation.scala

@@ -42,7 +42,8 @@ class DefaultFileBasedRelation(spark: SparkSession, override val plan: LogicalRe
   /**
    * Computes the signature of the current relation.
    */
-  override def signature: String = plan.relation match {
+  override def signature: String =
+    plan.relation match {
       case HadoopFsRelation(location: PartitioningAwareFileIndex, _, _, _, _, _) =>
         val result = filesFromIndex(location).sortBy(_.getPath.toString).foldLeft("") {
           (acc: String, f: FileStatus) =>
@@ -62,7 +63,8 @@ class DefaultFileBasedRelation(spark: SparkSession, override val plan: LogicalRe
   /**
    * The partition schema of the current relation.
    */
-  override def partitionSchema: StructType = plan.relation match {
+  override def partitionSchema: StructType =
+    plan.relation match {
       case HadoopFsRelation(location: FileIndex, _, _, _, _, _) =>
         location.partitionSchema
     }
@@ -70,7 +72,8 @@ class DefaultFileBasedRelation(spark: SparkSession, override val plan: LogicalRe
   /**
    * The optional partition base path of the current relation.
    */
-  override def partitionBasePath: Option[String] = plan.relation match {
+  override def partitionBasePath: Option[String] =
+    plan.relation match {
       case HadoopFsRelation(p: PartitioningAwareFileIndex, _, _, _, _, _)
           if p.partitionSpec.partitions.nonEmpty =>
         // For example, we could have the following in PartitionSpec:
@@ -93,7 +96,8 @@ class DefaultFileBasedRelation(spark: SparkSession, override val plan: LogicalRe
   override def createHadoopFsRelation(
       location: FileIndex,
       dataSchema: StructType,
-      options: Map[String, String]): HadoopFsRelation = plan.relation match {
+      options: Map[String, String]): HadoopFsRelation =
+    plan.relation match {
       case h: HadoopFsRelation =>
         h.copy(location = location, dataSchema = dataSchema, options = options)(spark)
     }
@@ -215,7 +219,8 @@ class DefaultFileBasedRelation(spark: SparkSession, override val plan: LogicalRe
    *
    * @return True if source files of the current relation are parquet.
    */
-  def hasParquetAsSourceFormat: Boolean = plan.relation match {
+  def hasParquetAsSourceFormat: Boolean =
+    plan.relation match {
       case h: HadoopFsRelation =>
         h.fileFormat.asInstanceOf[DataSourceRegister].shortName.equals("parquet")
     }

src/main/scala/com/microsoft/hyperspace/index/sources/default/DefaultFileBasedSource.scala

@@ -36,9 +36,11 @@ import com.microsoft.hyperspace.util.{CacheWithTransform, HyperspaceConf}
  */
 class DefaultFileBasedSource(private val spark: SparkSession) extends FileBasedSourceProvider {
   private val supportedFormats: CacheWithTransform[String, Set[String]] =
-    new CacheWithTransform[String, Set[String]]({ () =>
+    new CacheWithTransform[String, Set[String]](
+      { () =>
         HyperspaceConf.supportedFileFormatsForDefaultFileBasedSource(spark)
-    }, { formats =>
+      },
+      { formats =>
         formats.toLowerCase(Locale.ROOT).split(",").map(_.trim).toSet
       })
 
@@ -71,7 +73,8 @@ class DefaultFileBasedSource(private val spark: SparkSession) extends FileBasedS
    * @param plan Logical plan to check if it's supported.
    * @return Some(true) if the given plan is a supported relation, otherwise None.
    */
-  def isSupportedRelation(plan: LogicalPlan): Option[Boolean] = plan match {
+  def isSupportedRelation(plan: LogicalPlan): Option[Boolean] =
+    plan match {
       case LogicalRelation(
             HadoopFsRelation(_: PartitioningAwareFileIndex, _, _, _, fileFormat, _),
             _,

src/main/scala/com/microsoft/hyperspace/index/sources/delta/DeltaLakeFileBasedSource.scala

@@ -45,7 +45,8 @@ class DeltaLakeFileBasedSource(private val spark: SparkSession) extends FileBase
    * @param plan Logical plan to check if it's supported.
    * @return Some(true) if the given plan is a supported relation, otherwise None.
    */
-  def isSupportedRelation(plan: LogicalPlan): Option[Boolean] = plan match {
+  def isSupportedRelation(plan: LogicalPlan): Option[Boolean] =
+    plan match {
       case LogicalRelation(HadoopFsRelation(_: TahoeLogFileIndex, _, _, _, _, _), _, _, _) =>
         Some(true)
       case _ => None

src/main/scala/com/microsoft/hyperspace/index/sources/delta/DeltaLakeRelation.scala

@@ -37,7 +37,8 @@ class DeltaLakeRelation(spark: SparkSession, override val plan: LogicalRelation)
   /**
    * Computes the signature of the current relation.
    */
-  override def signature: String = plan.relation match {
+  override def signature: String =
+    plan.relation match {
       case HadoopFsRelation(location: TahoeLogFileIndex, _, _, _, _, _) =>
         location.tableVersion + location.path.toString
     }
@@ -57,7 +58,8 @@ class DeltaLakeRelation(spark: SparkSession, override val plan: LogicalRelation)
   /**
    * The optional partition base path of the current relation.
    */
-  override def partitionBasePath: Option[String] = plan.relation match {
+  override def partitionBasePath: Option[String] =
+    plan.relation match {
       case HadoopFsRelation(t: TahoeLogFileIndex, _, _, _, _, _) if t.partitionSchema.nonEmpty =>
         Some(t.path.toString)
       case _ => None

src/main/scala/com/microsoft/hyperspace/index/sources/delta/DeltaLakeRelationMetadata.scala

@@ -47,9 +47,12 @@ class DeltaLakeRelationMetadata(metadata: Relation) extends FileBasedRelationMet
     val deltaVerHistory = metadata.options.get("versionAsOf").map { deltaVersion =>
       val newVersionMapping = s"$indexVersion:$deltaVersion"
       DeltaLakeConstants.DELTA_VERSION_HISTORY_PROPERTY ->
-        properties.get(DeltaLakeConstants.DELTA_VERSION_HISTORY_PROPERTY).map { prop =>
+        properties
+          .get(DeltaLakeConstants.DELTA_VERSION_HISTORY_PROPERTY)
+          .map { prop =>
             s"$prop,$newVersionMapping"
-        }.getOrElse(newVersionMapping)
+          }
+          .getOrElse(newVersionMapping)
     }
     properties ++ deltaVerHistory
   }

src/main/scala/com/microsoft/hyperspace/util/ResolverUtils.scala

@@ -94,7 +94,9 @@ object ResolverUtils {
      */
     def apply(normalizedColumnName: String): ResolvedColumn = {
       if (normalizedColumnName.startsWith(NESTED_FIELD_PREFIX)) {
-        ResolvedColumn(normalizedColumnName.substring(NESTED_FIELD_PREFIX.length), isNested = true)
+        ResolvedColumn(
+          normalizedColumnName.substring(NESTED_FIELD_PREFIX.length),
+          isNested = true)
       } else {
         ResolvedColumn(normalizedColumnName, isNested = false)
       }
@@ -212,7 +214,8 @@ object ResolverUtils {
   private def getColumnNameFromSchema(
       schema: StructType,
       resolvedColNameParts: Seq[String],
-      resolver: Resolver): Seq[String] = resolvedColNameParts match {
+      resolver: Resolver): Seq[String] =
+    resolvedColNameParts match {
       case h :: tail =>
         val field = schema.find(f => resolver(f.name, h)).get
         field match {

src/test/scala/com/microsoft/hyperspace/SampleNestedData.scala

@@ -24,25 +24,75 @@ import org.apache.spark.sql.SparkSession
 object SampleNestedData {
 
   val testData = Seq(
-    ("2017-09-03", "810a20a2baa24ff3ad493bfbf064569a", "donde", 2, 1000,
+    (
+      "2017-09-03",
+      "810a20a2baa24ff3ad493bfbf064569a",
+      "donde",
+      2,
+      1000,
       SampleNestedDataStruct("id1", SampleNestedDataLeaf("leaf_id1", 1))),
-    ("2017-09-03", "fd093f8a05604515957083e70cb3dceb", "facebook", 1, 3000,
+    (
+      "2017-09-03",
+      "fd093f8a05604515957083e70cb3dceb",
+      "facebook",
+      1,
+      3000,
       SampleNestedDataStruct("id1", SampleNestedDataLeaf("leaf_id1", 2))),
-    ("2017-09-03", "af3ed6a197a8447cba8bc8ea21fad208", "facebook", 1, 3000,
+    (
+      "2017-09-03",
+      "af3ed6a197a8447cba8bc8ea21fad208",
+      "facebook",
+      1,
+      3000,
       SampleNestedDataStruct("id2", SampleNestedDataLeaf("leaf_id7", 1))),
-    ("2017-09-03", "975134eca06c4711a0406d0464cbe7d6", "facebook", 1, 4000,
+    (
+      "2017-09-03",
+      "975134eca06c4711a0406d0464cbe7d6",
+      "facebook",
+      1,
+      4000,
       SampleNestedDataStruct("id2", SampleNestedDataLeaf("leaf_id7", 2))),
-    ("2018-09-03", "e90a6028e15b4f4593eef557daf5166d", "ibraco", 2, 3000,
+    (
+      "2018-09-03",
+      "e90a6028e15b4f4593eef557daf5166d",
+      "ibraco",
+      2,
+      3000,
       SampleNestedDataStruct("id2", SampleNestedDataLeaf("leaf_id7", 5))),
-    ("2018-09-03", "576ed96b0d5340aa98a47de15c9f87ce", "facebook", 2, 3000,
+    (
+      "2018-09-03",
+      "576ed96b0d5340aa98a47de15c9f87ce",
+      "facebook",
+      2,
+      3000,
       SampleNestedDataStruct("id2", SampleNestedDataLeaf("leaf_id9", 1))),
-    ("2018-09-03", "50d690516ca641438166049a6303650c", "ibraco", 2, 1000,
+    (
+      "2018-09-03",
+      "50d690516ca641438166049a6303650c",
+      "ibraco",
+      2,
+      1000,
       SampleNestedDataStruct("id3", SampleNestedDataLeaf("leaf_id9", 10))),
-    ("2019-10-03", "380786e6495d4cd8a5dd4cc8d3d12917", "facebook", 2, 3000,
+    (
+      "2019-10-03",
+      "380786e6495d4cd8a5dd4cc8d3d12917",
+      "facebook",
+      2,
+      3000,
       SampleNestedDataStruct("id4", SampleNestedDataLeaf("leaf_id9", 12))),
-    ("2019-10-03", "ff60e4838b92421eafc3e6ee59a9e9f1", "miperro", 2, 2000,
+    (
+      "2019-10-03",
+      "ff60e4838b92421eafc3e6ee59a9e9f1",
+      "miperro",
+      2,
+      2000,
       SampleNestedDataStruct("id5", SampleNestedDataLeaf("leaf_id9", 21))),
-    ("2019-10-03", "187696fe0a6a40cc9516bc6e47c70bc1", "facebook", 4, 3000,
+    (
+      "2019-10-03",
+      "187696fe0a6a40cc9516bc6e47c70bc1",
+      "facebook",
+      4,
+      3000,
       SampleNestedDataStruct("id6", SampleNestedDataLeaf("leaf_id9", 22))))
 
   def save(
@@ -50,9 +100,7 @@ object SampleNestedData {
       path: String,
       columns: Seq[String],
       partitionColumns: Option[Seq[String]] = None): Unit = {
-    val df = spark.createDataFrame(
+    val df = spark.createDataFrame(spark.sparkContext.parallelize(testData)).toDF(columns: _*)
-      spark.sparkContext.parallelize(testData)
-    ).toDF(columns: _*)
     partitionColumns match {
       case Some(pcs) =>
         df.write.partitionBy(pcs: _*).parquet(path)

src/test/scala/com/microsoft/hyperspace/goldstandard/PlanStabilitySuite.scala

@@ -58,7 +58,6 @@ import com.microsoft.hyperspace.util.SparkTestShims.SimpleExplainCommand
  * The explain files are saved to help debug later, they are not checked. Only the simplified
  * plans are checked (by string comparison).
  *
- *
  * To run the entire test suite:
  * {{{
  *   sbt "test:testOnly *PlanStabilitySuite"
@@ -178,7 +177,8 @@ trait PlanStabilitySuite extends TPCDSBase with SQLHelper with Logging {
     val exchangeIdMap = new mutable.HashMap[SparkPlan, Int]()
     val subqueriesMap = new mutable.HashMap[SparkPlan, Int]()
 
-    def getId(plan: SparkPlan): Int = plan match {
+    def getId(plan: SparkPlan): Int =
+      plan match {
         case exchange: Exchange => exchangeIdMap.getOrElseUpdate(exchange, exchangeIdMap.size + 1)
         case ReusedExchangeExec(_, exchange) =>
           exchangeIdMap.getOrElseUpdate(exchange, exchangeIdMap.size + 1)
@@ -273,7 +273,9 @@ trait PlanStabilitySuite extends TPCDSBase with SQLHelper with Logging {
  */
 class TPCDSV1_4_SparkPlanStabilitySuite extends PlanStabilitySuite {
   override val goldenFilePath: String = {
-    new File(baseResourcePath, s"spark-${BuildInfo.sparkShortVersion}/approved-plans-v1_4").getAbsolutePath
+    new File(
+      baseResourcePath,
+      s"spark-${BuildInfo.sparkShortVersion}/approved-plans-v1_4").getAbsolutePath
   }
 
   // Enable cross join because some queries fail during query optimization phase.

src/test/scala/com/microsoft/hyperspace/goldstandard/TPCDSBase.scala

@@ -536,8 +536,7 @@ trait TPCDSBase extends SparkFunSuite with SparkInvolvedSuite {
         |`t_shift` STRING,
         |`t_sub_shift` STRING,
         |`t_meal_time` STRING
-      """.stripMargin
+      """.stripMargin)
-  )
 
   val tableNames: Iterable[String] = tableColumns.keys
 
@@ -546,8 +545,7 @@ trait TPCDSBase extends SparkFunSuite with SparkInvolvedSuite {
       tableName: String,
       format: String = "parquet",
       options: Seq[String] = Nil): Unit = {
-    spark.sql(
+    spark.sql(s"""
-      s"""
          |CREATE TABLE `$tableName` (${tableColumns(tableName)})
          |USING $format
          |${options.mkString("\n")}

src/test/scala/com/microsoft/hyperspace/index/CreateIndexNestedTest.scala

@@ -141,10 +141,7 @@ class CreateIndexNestedTest extends HyperspaceSuite with SQLHelper {
     val dfB = nonPartitionedDataDF.as("B")
     val dfJoin = dfA
       .join(dfB, dfA("Query") === dfB("Query"))
-      .select(
+      .select(dfA("RGUID"), dfA("Query"), dfA("nested.leaf.cnt"))
-        dfA("RGUID"),
-        dfA("Query"),
-        dfA("nested.leaf.cnt"))
     val exception = intercept[HyperspaceException] {
       hyperspace.createIndex(dfJoin, indexConfig1)
     }
@@ -153,7 +150,8 @@ class CreateIndexNestedTest extends HyperspaceSuite with SQLHelper {
         "Only creating index over HDFS file based scan nodes is supported."))
   }
 
-  test("Check lineage in index records for partitioned data when partition key is not in config.") {
+  test(
+    "Check lineage in index records for partitioned data when partition key is not in config.") {
     withSQLConf(IndexConstants.INDEX_LINEAGE_ENABLED -> "true") {
       hyperspace.createIndex(partitionedDataDF, indexConfig2)
       val indexRecordsDF = spark.read.parquet(

src/test/scala/com/microsoft/hyperspace/index/CreateIndexTest.scala

@@ -145,10 +145,7 @@ class CreateIndexTest extends HyperspaceSuite with SQLHelper {
     val dfB = nonPartitionedDataDF.as("B")
     val dfJoin = dfA
       .join(dfB, dfA("Query") === dfB("Query"))
-      .select(
+      .select(dfA("RGUID"), dfA("Query"), dfA("imprs"))
-        dfA("RGUID"),
-        dfA("Query"),
-        dfA("imprs"))
     val exception = intercept[HyperspaceException] {
       hyperspace.createIndex(dfJoin, indexConfig1)
     }
@@ -171,7 +168,8 @@ class CreateIndexTest extends HyperspaceSuite with SQLHelper {
     }
   }
 
-  test("Check lineage in index records for partitioned data when partition key is not in config.") {
+  test(
+    "Check lineage in index records for partitioned data when partition key is not in config.") {
     withSQLConf(IndexConstants.INDEX_LINEAGE_ENABLED -> "true") {
       hyperspace.createIndex(partitionedDataDF, indexConfig3)
       val indexRecordsDF = spark.read.parquet(
@@ -201,7 +199,8 @@ class CreateIndexTest extends HyperspaceSuite with SQLHelper {
     }
   }
 
-  test("Check lineage in index records for partitioned data when partition key is in load path.") {
+  test(
+    "Check lineage in index records for partitioned data when partition key is in load path.") {
     withSQLConf(IndexConstants.INDEX_LINEAGE_ENABLED -> "true") {
       val dataDF =
         spark.read.parquet(s"$partitionedDataPath/${partitionKeys.head}=2017-09-03")

src/test/scala/com/microsoft/hyperspace/index/E2EHyperspaceRulesTest.scala

@@ -435,7 +435,8 @@ class E2EHyperspaceRulesTest extends QueryTest with HyperspaceSuite {
       sortedRowsWithHyperspaceDisabled.sameElements(getSortedRows(dfAfterHyperspaceDisabled)))
   }
 
-  test("Verify JoinIndexRule utilizes indexes correctly after incremental refresh (append-only).") {
+  test(
+    "Verify JoinIndexRule utilizes indexes correctly after incremental refresh (append-only).") {
     withTempPathAsString { testPath =>
       // Setup. Create data.
       val indexConfig = IndexConfig("index", Seq("c2"), Seq("c4"))
@@ -596,7 +597,8 @@ class E2EHyperspaceRulesTest extends QueryTest with HyperspaceSuite {
         verifyIndexUsage(
           query,
           getIndexFilesPath(indexConfig.indexName, Seq(1)) ++ // for Left
-            getIndexFilesPath(indexConfig.indexName, Seq(1))) // for Right
+            getIndexFilesPath(indexConfig.indexName, Seq(1))
+        ) // for Right
 
         // Verify correctness of results.
         spark.disableHyperspace()
@@ -659,7 +661,8 @@ class E2EHyperspaceRulesTest extends QueryTest with HyperspaceSuite {
         verifyIndexUsage(
           query,
           getIndexFilesPath(indexConfig.indexName, Seq(0)) ++ appendedFiles ++
-            getIndexFilesPath(indexConfig.indexName, Seq(0)) ++ appendedFiles) // for Right
+            getIndexFilesPath(indexConfig.indexName, Seq(0)) ++ appendedFiles
+        ) // for Right
 
         // Verify correctness of results.
         spark.disableHyperspace()

src/test/scala/com/microsoft/hyperspace/index/FileIdTrackerTest.scala

@@ -58,14 +58,15 @@ class FileIdTrackerTest extends SparkFunSuite {
     val tracker = new FileIdTracker
     tracker.addFileInfo(Set(FileInfo("def", 123, 555, 10)))
     val ex = intercept[HyperspaceException] {
-      implicit def ordering: Ordering[FileInfo] = new Ordering[FileInfo] {
+      implicit def ordering: Ordering[FileInfo] =
+        new Ordering[FileInfo] {
           override def compare(x: FileInfo, y: FileInfo): Int = {
             x.name.compareTo(y.name)
           }
         }
-      tracker.addFileInfo(scala.collection.immutable.SortedSet(
+      tracker.addFileInfo(
-        FileInfo("abc", 100, 555, 15),
+        scala.collection.immutable
-        FileInfo("def", 123, 555, 11)))
+          .SortedSet(FileInfo("abc", 100, 555, 15), FileInfo("def", 123, 555, 11)))
     }
     assert(ex.getMessage.contains("Adding file info with a conflicting id"))
     assert(tracker.getFileId("abc", 100, 555).contains(15))

src/test/scala/com/microsoft/hyperspace/index/HybridScanForNonPartitionedDataTest.scala

@@ -142,13 +142,15 @@ class HybridScanForNonPartitionedDataTest extends HybridScanSuite {
       val deletedRatio = 1 - (afterDeleteSize / sourceSize.toFloat)
 
       withSQLConf(TestConfig.HybridScanEnabled: _*) {
-        withSQLConf(IndexConstants.INDEX_HYBRID_SCAN_DELETED_RATIO_THRESHOLD ->
+        withSQLConf(
+          IndexConstants.INDEX_HYBRID_SCAN_DELETED_RATIO_THRESHOLD ->
             (deletedRatio + 0.1).toString) {
           val filter = filterQuery
           // As deletedRatio is less than the threshold, the index can be applied.
           assert(!basePlan.equals(filter.queryExecution.optimizedPlan))
         }
-        withSQLConf(IndexConstants.INDEX_HYBRID_SCAN_DELETED_RATIO_THRESHOLD ->
+        withSQLConf(
+          IndexConstants.INDEX_HYBRID_SCAN_DELETED_RATIO_THRESHOLD ->
             (deletedRatio - 0.1).toString) {
           val filter = filterQuery
           // As deletedRatio is greater than the threshold, the index shouldn't be applied.

src/test/scala/com/microsoft/hyperspace/index/HybridScanSuite.scala

@@ -290,7 +290,8 @@ trait HybridScanSuite extends QueryTest with HyperspaceSuite {
         case p @ BucketUnionExec(children, bucketSpec) =>
           assert(children.size === 2)
           // children.head is always the index plan.
-          assert(children.head.isInstanceOf[ProjectExec] || children.head.isInstanceOf[FilterExec])
+          assert(
+            children.head.isInstanceOf[ProjectExec] || children.head.isInstanceOf[FilterExec])
           assert(children.last.isInstanceOf[ShuffleExchangeExec])
           assert(bucketSpec.numBuckets === 200)
           p
@@ -582,7 +583,8 @@ trait HybridScanSuite extends QueryTest with HyperspaceSuite {
     }
   }
 
-  test("Delete-only: join rule, deleted files should be excluded from each index data relation.") {
+  test(
+    "Delete-only: join rule, deleted files should be excluded from each index data relation.") {
     withTempPathAsString { testPath =>
       val deletePath1 = testPath + "/delete1"
       val deletePath2 = testPath + "/delete2"

src/test/scala/com/microsoft/hyperspace/index/IndexCacheTest.scala

@@ -48,12 +48,7 @@ class IndexCacheTest extends HyperspaceSuite {
 
     val entry = IndexLogEntry(
       "index1",
-      CoveringIndex(
+      CoveringIndex(Seq("RGUID"), Seq("Date"), schema, 10, Map()),
-        Seq("RGUID"),
-        Seq("Date"),
-        schema,
-        10,
-        Map()),
       Content(Directory(indexDir)),
       Source(SparkPlan(sourcePlanProperties)),
       Map())
@@ -152,7 +147,7 @@ class IndexCacheTest extends HyperspaceSuite {
 
 /**
  * Mock for testing purposes so we can validate and invalidate entries based on time.
-
+ *
  * @param time Current time.
  */
 class MockClock(private var time: Long = 0L) extends Clock {

src/test/scala/com/microsoft/hyperspace/index/IndexCollectionManagerTest.scala

@@ -47,12 +47,7 @@ class IndexCollectionManagerTest extends HyperspaceSuite {
 
           val entry = IndexLogEntry(
             indexPath.toString,
-            CoveringIndex(
+            CoveringIndex(Seq("RGUID"), Seq("Date"), new StructType(), 10, Map()),
-              Seq("RGUID"),
-              Seq("Date"),
-              new StructType(),
-              10,
-              Map()),
             Content(Directory(s"$indexPath/${IndexConstants.INDEX_VERSION_DIRECTORY_PREFIX}=0")),
             Source(SparkPlan(sourcePlanProperties)),
             Map())
@@ -98,12 +93,7 @@ class IndexCollectionManagerTest extends HyperspaceSuite {
 
       val entry = IndexLogEntry(
         str,
-        CoveringIndex(
+        CoveringIndex(Seq("RGUID"), Seq("Date"), new StructType(), 10, Map()),
-          Seq("RGUID"),
-          Seq("Date"),
-          new StructType(),
-          10,
-          Map()),
         Content(Directory(s"$str/${IndexConstants.INDEX_VERSION_DIRECTORY_PREFIX}=0")),
         Source(SparkPlan(sourcePlanProperties)),
         Map())

src/test/scala/com/microsoft/hyperspace/index/IndexLogEntryTest.scala

@@ -193,43 +193,24 @@ class IndexLogEntryTest extends HyperspaceSuite with SQLHelper {
       Seq(
         Relation(
           Seq("rootpath"),
-          Hdfs(
+          Hdfs(Hdfs.Properties(
-            Hdfs.Properties(
+            Content(Directory(
-              Content(
-                Directory(
               "test",
               Seq(FileInfo("f1", 100L, 100L, 0), FileInfo("f2", 100L, 200L, 1)),
-                  Seq()
+              Seq())),
-                )
+            Some(Update(None, Some(Content(Directory("", Seq(FileInfo("f1", 10, 10, 2))))))))),
-              ),
-              Some(
-                Update(
-                  None,
-                  Some(Content(Directory("", Seq(FileInfo("f1", 10, 10, 2)))))
-                )
-              )
-            )
-          ),
           new StructType(),
           "type",
-          Map()
+          Map())),
-        )
-      ),
       null,
       null,
       LogicalPlanFingerprint(
         LogicalPlanFingerprint
-          .Properties(Seq(Signature("provider", "signatureValue")))
+          .Properties(Seq(Signature("provider", "signatureValue")))))
-      ))
 
     val expected = IndexLogEntry.create(
       "indexName",
-      CoveringIndex(
+      CoveringIndex(Seq("col1"), Seq("col2", "col3"), schema, 200, Map()),
-        Seq("col1"),
-        Seq("col2", "col3"),
-        schema,
-        200,
-        Map()),
       Content(Directory("rootContentPath")),
       Source(SparkPlan(expectedSourcePlanProperties)),
       Map())
@@ -242,15 +223,19 @@ class IndexLogEntryTest extends HyperspaceSuite with SQLHelper {
   }
 
   test("Content.files api lists all files from Content object.") {
-    val content = Content(Directory("file:/", subDirs =
+    val content = Content(
-      Seq(
+      Directory(
-        Directory("a",
+        "file:/",
-          files = Seq(FileInfo("f1", 0, 0, UNKNOWN_FILE_ID), FileInfo("f2", 0, 0, UNKNOWN_FILE_ID)),
+        subDirs = Seq(Directory(
-          subDirs = Seq(
+          "a",
-            Directory("b",
           files =
-                Seq(FileInfo("f3", 0, 0, UNKNOWN_FILE_ID), FileInfo("f4", 0, 0, UNKNOWN_FILE_ID)))))
+            Seq(FileInfo("f1", 0, 0, UNKNOWN_FILE_ID), FileInfo("f2", 0, 0, UNKNOWN_FILE_ID)),
-      )))
+          subDirs = Seq(
+            Directory(
+              "b",
+              files = Seq(
+                FileInfo("f3", 0, 0, UNKNOWN_FILE_ID),
+                FileInfo("f4", 0, 0, UNKNOWN_FILE_ID))))))))
 
     val expected =
       Seq("file:/a/f1", "file:/a/f2", "file:/a/b/f3", "file:/a/b/f4").map(new Path(_)).toSet
@@ -262,8 +247,9 @@ class IndexLogEntryTest extends HyperspaceSuite with SQLHelper {
     val nestedDirPath = toPath(nestedDir)
 
     val expected = {
-      val fileInfos = Seq(f3, f4).map(toFileStatus).map(f =>
+      val fileInfos = Seq(f3, f4)
-        FileInfo(f, fileIdTracker.addFile(f), asFullPath = false))
+        .map(toFileStatus)
+        .map(f => FileInfo(f, fileIdTracker.addFile(f), asFullPath = false))
       val nestedDirDirectory = Directory("nested", fileInfos)
       val rootDirectory = createDirectory(nestedDirPath, nestedDirDirectory)
       Content(rootDirectory, NoOpFingerprint())
@@ -277,8 +263,9 @@ class IndexLogEntryTest extends HyperspaceSuite with SQLHelper {
     val nestedDirPath = toPath(nestedDir)
 
     val expected = {
-      val fileInfos = Seq(f3, f4).map(toFileStatus).map(f =>
+      val fileInfos = Seq(f3, f4)
-        FileInfo(f, fileIdTracker.addFile(f), asFullPath = false))
+        .map(toFileStatus)
+        .map(f => FileInfo(f, fileIdTracker.addFile(f), asFullPath = false))
       val nestedDirDirectory = Directory("nested", fileInfos)
       val rootDirectory = createDirectory(nestedDirPath, nestedDirDirectory)
       Content(rootDirectory, NoOpFingerprint())
@@ -292,8 +279,9 @@ class IndexLogEntryTest extends HyperspaceSuite with SQLHelper {
     val nestedDirPath = toPath(nestedDir)
 
     val expected = {
-      val fileInfos = Seq(f3, f4).map(toFileStatus).map(f =>
+      val fileInfos = Seq(f3, f4)
-        FileInfo(f, fileIdTracker.addFile(f), asFullPath = false))
+        .map(toFileStatus)
+        .map(f => FileInfo(f, fileIdTracker.addFile(f), asFullPath = false))
       val nestedDirDirectory = Directory("nested", fileInfos)
       createDirectory(nestedDirPath, nestedDirDirectory)
     }
@@ -302,17 +290,21 @@ class IndexLogEntryTest extends HyperspaceSuite with SQLHelper {
     assert(directoryEquals(actual, expected))
   }
 
-  test("Directory.fromDirectory api creates the correct Directory objects, " +
+  test(
+    "Directory.fromDirectory api creates the correct Directory objects, " +
       "recursively listing all leaf files.") {
     val testDirPath = toPath(testDir)
 
     val testDirLeafFiles =
-      Seq(f1, f2).map(toFileStatus).map(f =>
+      Seq(f1, f2)
-        FileInfo(f, fileIdTracker.addFile(f), asFullPath = false))
+        .map(toFileStatus)
+        .map(f => FileInfo(f, fileIdTracker.addFile(f), asFullPath = false))
     val nestedDirLeafFiles =
-      Seq(f3, f4).map(toFileStatus).map(f =>
+      Seq(f3, f4)
-        FileInfo(f, fileIdTracker.addFile(f), asFullPath = false))
+        .map(toFileStatus)
-    val testDirDirectory = Directory(name = "testDir",
+        .map(f => FileInfo(f, fileIdTracker.addFile(f), asFullPath = false))
+    val testDirDirectory = Directory(
+      name = "testDir",
       files = testDirLeafFiles,
       subDirs = Seq(Directory(name = "nested", files = nestedDirLeafFiles)))
     val expected = createDirectory(testDirPath, testDirDirectory)
@@ -326,12 +318,15 @@ class IndexLogEntryTest extends HyperspaceSuite with SQLHelper {
     val testDirPath = toPath(testDir)
 
     val testDirLeafFiles =
-      Seq(f1, f2).map(toFileStatus).map(f =>
+      Seq(f1, f2)
-        FileInfo(f, fileIdTracker.addFile(f), asFullPath = false))
+        .map(toFileStatus)
+        .map(f => FileInfo(f, fileIdTracker.addFile(f), asFullPath = false))
     val nestedDirLeafFiles =
-      Seq(f3, f4).map(toFileStatus).map(f =>
+      Seq(f3, f4)
-        FileInfo(f, fileIdTracker.addFile(f), asFullPath = false))
+        .map(toFileStatus)
-    val testDirDirectory = Directory(name = "testDir",
+        .map(f => FileInfo(f, fileIdTracker.addFile(f), asFullPath = false))
+    val testDirDirectory = Directory(
+      name = "testDir",
       files = testDirLeafFiles,
       subDirs = Seq(Directory(name = "nested", files = nestedDirLeafFiles)))
 
@@ -345,12 +340,15 @@ class IndexLogEntryTest extends HyperspaceSuite with SQLHelper {
   test("Directory.fromLeafFiles api does not include other files in the directory.") {
     val testDirPath = toPath(testDir)
 
-    val testDirLeafFiles = Seq(f1).map(toFileStatus).map(f =>
+    val testDirLeafFiles = Seq(f1)
-      FileInfo(f, fileIdTracker.addFile(f), asFullPath = false))
+      .map(toFileStatus)
+      .map(f => FileInfo(f, fileIdTracker.addFile(f), asFullPath = false))
     val nestedDirLeafFiles =
-      Seq(f4).map(toFileStatus).map(f =>
+      Seq(f4)
-        FileInfo(f, fileIdTracker.addFile(f), asFullPath = false))
+        .map(toFileStatus)
-    val testDirDirectory = Directory(name = "testDir",
+        .map(f => FileInfo(f, fileIdTracker.addFile(f), asFullPath = false))
+    val testDirDirectory = Directory(
+      name = "testDir",
       files = testDirLeafFiles,
       subDirs = Seq(Directory(name = "nested", files = nestedDirLeafFiles)))
 
@@ -389,8 +387,9 @@ class IndexLogEntryTest extends HyperspaceSuite with SQLHelper {
       override def accept(path: Path): Boolean = path.getName.startsWith("f1")
     }
 
-    val testDirLeafFiles = Seq(f1).map(toFileStatus).map(f =>
+    val testDirLeafFiles = Seq(f1)
-      FileInfo(f, fileIdTracker.addFile(f), asFullPath = false))
+      .map(toFileStatus)
+      .map(f => FileInfo(f, fileIdTracker.addFile(f), asFullPath = false))
     val testDirDirectory = Directory(name = "testDir", files = testDirLeafFiles)
     val expected = createDirectory(testDirPath, testDirDirectory)
 
@@ -400,7 +399,8 @@ class IndexLogEntryTest extends HyperspaceSuite with SQLHelper {
     assert(directoryEquals(actual, expected))
   }
 
-  test("Directory.fromDirectory and fromLeafFileswhere files are at same level but different" +
+  test(
+    "Directory.fromDirectory and fromLeafFileswhere files are at same level but different" +
       "dirs.") {
     // File Structure
     // testDir/temp/a/f1
@@ -413,11 +413,17 @@ class IndexLogEntryTest extends HyperspaceSuite with SQLHelper {
     val f2 = Files.createFile(Paths.get(b + "/f2"))
 
     val aDirectory =
-      Directory("a", Seq(f1).map(toFileStatus).map(f =>
+      Directory(
-        FileInfo(f, fileIdTracker.addFile(f), asFullPath = false)))
+        "a",
+        Seq(f1)
+          .map(toFileStatus)
+          .map(f => FileInfo(f, fileIdTracker.addFile(f), asFullPath = false)))
     val bDirectory =
-      Directory("b", Seq(f2).map(toFileStatus).map(f =>
+      Directory(
-        FileInfo(f, fileIdTracker.addFile(f), asFullPath = false)))
+        "b",
+        Seq(f2)
+          .map(toFileStatus)
+          .map(f => FileInfo(f, fileIdTracker.addFile(f), asFullPath = false)))
     val tempDirectory = Directory("temp", subDirs = Seq(aDirectory, bDirectory))
     val tempDirectoryPath = toPath(tempDir)
 
@@ -445,12 +451,18 @@ class IndexLogEntryTest extends HyperspaceSuite with SQLHelper {
     val f2 = Files.createFile(Paths.get(c + "/f2"))
 
     val cDirectory =
-      Directory("c", Seq(f2).map(toFileStatus).map(f =>
+      Directory(
-        FileInfo(f, fileIdTracker.addFile(f), asFullPath = false)))
+        "c",
+        Seq(f2)
+          .map(toFileStatus)
+          .map(f => FileInfo(f, fileIdTracker.addFile(f), asFullPath = false)))
     val bDirectory = Directory("b", subDirs = Seq(cDirectory))
     val aDirectory =
-      Directory("a", Seq(f1).map(toFileStatus).map(f =>
+      Directory(
-        FileInfo(f, fileIdTracker.addFile(f), asFullPath = false)))
+        "a",
+        Seq(f1)
+          .map(toFileStatus)
+          .map(f => FileInfo(f, fileIdTracker.addFile(f), asFullPath = false)))
 
     val tempDirectory = Directory("temp", subDirs = Seq(aDirectory, bDirectory))
     val tempDirectoryPath = toPath(tempDir)
@@ -465,7 +477,8 @@ class IndexLogEntryTest extends HyperspaceSuite with SQLHelper {
     FileUtils.deleteDirectory(tempDir.toFile)
   }
 
-  test("Directory.fromDirectory and fromLeafFiles where files belong to multiple" +
+  test(
+    "Directory.fromDirectory and fromLeafFiles where files belong to multiple" +
       "subdirectories.") {
     // File Structure
     // testDir/temp/a/f1
@@ -481,17 +494,23 @@ class IndexLogEntryTest extends HyperspaceSuite with SQLHelper {
     val f3 = Files.createFile(Paths.get(c + "/f3"))
 
     val bDirectory =
-      Directory("b", Seq(f2).map(toFileStatus).map(f =>
+      Directory(
-        FileInfo(f, fileIdTracker.addFile(f), asFullPath = false)))
+        "b",
+        Seq(f2)
+          .map(toFileStatus)
+          .map(f => FileInfo(f, fileIdTracker.addFile(f), asFullPath = false)))
     val cDirectory =
-      Directory("c", Seq(f3).map(toFileStatus).map(f =>
+      Directory(
-        FileInfo(f, fileIdTracker.addFile(f), asFullPath = false)))
+        "c",
+        Seq(f3)
+          .map(toFileStatus)
+          .map(f => FileInfo(f, fileIdTracker.addFile(f), asFullPath = false)))
     val aDirectory = Directory(
       "a",
-      Seq(f1).map(toFileStatus).map(f =>
+      Seq(f1)
-        FileInfo(f, fileIdTracker.addFile(f), asFullPath = false)),
+        .map(toFileStatus)
-      Seq(bDirectory, cDirectory)
+        .map(f => FileInfo(f, fileIdTracker.addFile(f), asFullPath = false)),
-    )
+      Seq(bDirectory, cDirectory))
     val tempDirectory = Directory("temp", subDirs = Seq(aDirectory))
     val tempDirectoryPath = toPath(tempDir)
 
@@ -511,11 +530,7 @@ class IndexLogEntryTest extends HyperspaceSuite with SQLHelper {
     // a/f2
     val directory1 = Directory(
       name = "a",
-      files = Seq(
+      files = Seq(FileInfo("f1", 100L, 100L, 1L), FileInfo("f2", 100L, 100L, 2L)))
-        FileInfo("f1", 100L, 100L, 1L),
-        FileInfo("f2", 100L, 100L, 2L)
-      )
-    )
 
     // directory2:
     // a/b/f3
@@ -525,13 +540,7 @@ class IndexLogEntryTest extends HyperspaceSuite with SQLHelper {
       subDirs = Seq(
         Directory(
           name = "b",
-          files = Seq(
+          files = Seq(FileInfo("f3", 100L, 100L, 3L), FileInfo("f4", 100L, 100L, 4L)))))
-            FileInfo("f3", 100L, 100L, 3L),
-            FileInfo("f4", 100L, 100L, 4L)
-          )
-        )
-      )
-    )
 
     // Expected result of merging directory1 and directory2:
     // a/f1
@@ -540,20 +549,11 @@ class IndexLogEntryTest extends HyperspaceSuite with SQLHelper {
     // a/b/f4
     val expected = Directory(
       name = "a",
-      files = Seq(
+      files = Seq(FileInfo("f1", 100L, 100L, 1L), FileInfo("f2", 100L, 100L, 2L)),
-        FileInfo("f1", 100L, 100L, 1L),
-        FileInfo("f2", 100L, 100L, 2L)
-      ),
       subDirs = Seq(
         Directory(
           name = "b",
-          files = Seq(
+          files = Seq(FileInfo("f3", 100L, 100L, 3L), FileInfo("f4", 100L, 100L, 4L)))))
-            FileInfo("f3", 100L, 100L, 3L),
-            FileInfo("f4", 100L, 100L, 4L)
-          )
-        )
-      )
-    )
 
     val actual1 = directory1.merge(directory2)
     val actual2 = directory2.merge(directory1)
@@ -569,14 +569,8 @@ class IndexLogEntryTest extends HyperspaceSuite with SQLHelper {
     // a/b/f3
     val directory1 = Directory(
       name = "a",
-      files = Seq(
+      files = Seq(FileInfo("f1", 100L, 100L, 1L), FileInfo("f2", 100L, 100L, 2L)),
-        FileInfo("f1", 100L, 100L, 1L),
+      subDirs = Seq(Directory(name = "b", files = Seq(FileInfo("f3", 100L, 100L, 3L)))))
-        FileInfo("f2", 100L, 100L, 2L)
-      ),
-      subDirs = Seq(
-        Directory(name = "b", files = Seq(FileInfo("f3", 100L, 100L, 3L)))
-      )
-    )
 
     // directory2:
     // a/f4
@@ -589,17 +583,8 @@ class IndexLogEntryTest extends HyperspaceSuite with SQLHelper {
       subDirs = Seq(
         Directory(
           name = "b",
-          files = Seq(
+          files = Seq(FileInfo("f5", 100L, 100L, 5L), FileInfo("f6", 100L, 100L, 6L)),
-            FileInfo("f5", 100L, 100L, 5L),
+          subDirs = Seq(Directory(name = "c", files = Seq(FileInfo("f7", 100L, 100L, 7L)))))))
-            FileInfo("f6", 100L, 100L, 6L)
-          ),
-          subDirs = Seq(Directory(
-            name = "c",
-            files = Seq(FileInfo("f7", 100L, 100L, 7L))
-          ))
-        )
-      )
-    )
 
     // Expected result of merging directory1 and directory2:
     // directory1:
@@ -615,23 +600,15 @@ class IndexLogEntryTest extends HyperspaceSuite with SQLHelper {
       files = Seq(
         FileInfo("f1", 100L, 100L, 1L),
         FileInfo("f2", 100L, 100L, 2L),
-        FileInfo("f4", 100L, 100L, 4L)
+        FileInfo("f4", 100L, 100L, 4L)),
-      ),
       subDirs = Seq(
         Directory(
           name = "b",
           files = Seq(
             FileInfo("f3", 100L, 100L, 3L),
             FileInfo("f5", 100L, 100L, 5L),
-            FileInfo("f6", 100L, 100L, 6L)
+            FileInfo("f6", 100L, 100L, 6L)),
-          ),
+          subDirs = Seq(Directory("c", files = Seq(FileInfo("f7", 100L, 100L, 7L)))))))
-          subDirs = Seq(
-            Directory("c",
-              files = Seq(FileInfo("f7", 100L, 100L, 7L)))
-          )
-        )
-      )
-    )
 
     val actual1 = directory1.merge(directory2)
     val actual2 = directory2.merge(directory1)
@@ -646,16 +623,14 @@ class IndexLogEntryTest extends HyperspaceSuite with SQLHelper {
     // a/f2
     val directory1 = Directory(
       name = "a",
-      files = Seq(FileInfo("f1", 100L, 100L, 1L), FileInfo("f2", 100L, 100L, 2L))
+      files = Seq(FileInfo("f1", 100L, 100L, 1L), FileInfo("f2", 100L, 100L, 2L)))
-    )
 
     // directory2:
     // b/f3
     // b/f4
     val directory2 = Directory(
       name = "b",
-      files = Seq(FileInfo("f3", 100L, 100L, 3L), FileInfo("f4", 100L, 100L, 4L))
+      files = Seq(FileInfo("f3", 100L, 100L, 3L), FileInfo("f4", 100L, 100L, 4L)))
-    )
 
     val ex1 = intercept[HyperspaceException](directory1.merge(directory2))
     val ex2 = intercept[HyperspaceException](directory2.merge(directory1))
@@ -680,8 +655,7 @@ class IndexLogEntryTest extends HyperspaceSuite with SQLHelper {
   // Using `directoryPath`, create a Directory tree starting from root and ending at
   // `leafDirectory`.
   private def createDirectory(directoryPath: Path, leafDirectory: Directory): Directory = {
-    TestUtils.splitPath(directoryPath.getParent).foldLeft(leafDirectory) {
+    TestUtils.splitPath(directoryPath.getParent).foldLeft(leafDirectory) { (accum, name) =>
-      (accum, name) =>
       Directory(name, Seq(), Seq(accum))
     }
   }

src/test/scala/com/microsoft/hyperspace/index/IndexTest.scala

@@ -39,12 +39,7 @@ class IndexTest extends SparkFunSuite {
 
     val entry = IndexLogEntry(
       config.indexName,
-      CoveringIndex(
+      CoveringIndex(config.indexedColumns, config.includedColumns, schema, numBuckets, Map()),
-        config.indexedColumns,
-        config.includedColumns,
-        schema,
-        numBuckets,
-        Map()),
       Content(Directory(path)),
       Source(SparkPlan(sourcePlanProperties)),
       Map())

src/test/scala/com/microsoft/hyperspace/index/rankers/JoinIndexRankerTest.scala

@@ -116,7 +116,6 @@ class JoinIndexRankerTest extends HyperspaceRuleSuite with SQLHelper {
       val r_20 = createIndexLogEntry("r2", Seq(t2c1), Seq(t2c2), rightPlan, 20, fileList1, false)
       setCommonSourceSizeInBytesTag(r_20, rightPlan, fileList1)
 
-
       val indexPairs = Seq((l_10, r_10), (l_10, r_20), (l_20, r_20))
       val expectedOrder = Seq((l_20, r_20), (l_10, r_10), (l_10, r_20))
       val actualOrder = JoinIndexRanker.rank(spark, leftPlan, rightPlan, indexPairs)

src/test/scala/com/microsoft/hyperspace/index/rules/FilterIndexRuleTest.scala

@@ -138,7 +138,8 @@ class FilterIndexRuleTest extends HyperspaceRuleSuite {
     }
   }
 
-  test("Verify FilterIndex rule does not apply if filter does not contain first indexed column.") {
+  test(
+    "Verify FilterIndex rule does not apply if filter does not contain first indexed column.") {
     val filterCondition =
       And(IsNotNull(c2), EqualTo(c2, Literal("RGUID_VALUE"))) // c2 is not first indexed column
     val filterNode = Filter(filterCondition, scanNode)

src/test/scala/com/microsoft/hyperspace/index/rules/JoinIndexRuleTest.scala

@@ -259,7 +259,8 @@ class JoinIndexRuleTest extends HyperspaceRuleSuite with SQLHelper {
     }
   }
 
-  test("Join rule does not update plan if index doesn't satisfy included columns from any side.") {
+  test(
+    "Join rule does not update plan if index doesn't satisfy included columns from any side.") {
     val t1FilterNode = Filter(IsNotNull(t1c1), t1ScanNode)
     val t2FilterNode = Filter(IsNotNull(t2c1), t2ScanNode)
 

src/test/scala/com/microsoft/hyperspace/util/JsonUtilsTest.scala

@@ -39,12 +39,7 @@ class JsonUtilsTest extends SparkFunSuite {
 
     val index = IndexLogEntry(
       "myIndex",
-      CoveringIndex(
+      CoveringIndex(Seq("id"), Seq("name", "school"), schema, 10, Map()),
-        Seq("id"),
-        Seq("name", "school"),
-        schema,
-        10,
-        Map()),
       Content(Directory("path")),
       Source(SparkPlan(sourcePlanProperties)),
       Map())