#146 Add S3 support for files with fixed-length records.

AbsaOSS · Dec 28, 2020 · 12f4aa9 · 12f4aa9
1 parent 60f88db
commit 12f4aa9
Show file tree

Hide file tree

Showing 3 changed files with 26 additions and 2 deletions.
diff --git a/spark-cobol/src/main/scala/za/co/absa/cobrix/spark/cobol/source/scanners/CobolScanners.scala b/spark-cobol/src/main/scala/za/co/absa/cobrix/spark/cobol/source/scanners/CobolScanners.scala
@@ -111,7 +111,7 @@ private[source] object CobolScanners {
 
   private def areThereNonDivisibleFiles(sourceDir: String, hadoopConfiguration: Configuration, divisor: Int): Boolean = {
 
-    val fileSystem = FileSystem.get(hadoopConfiguration)
+    val fileSystem = new Path(sourceDir).getFileSystem(hadoopConfiguration)
 
     if (FileUtils.getNumberOfFilesInDir(sourceDir, fileSystem) < FileUtils.THRESHOLD_DIR_LENGTH_FOR_SINGLE_FILE_CHECK) {
       FileUtils.findAndLogAllNonDivisibleFiles(sourceDir, divisor, fileSystem) > 0

diff --git a/spark-cobol/src/main/scala/za/co/absa/cobrix/spark/cobol/utils/FileUtils.scala b/spark-cobol/src/main/scala/za/co/absa/cobrix/spark/cobol/utils/FileUtils.scala
@@ -52,7 +52,7 @@ object FileUtils {
     * The directory may be informed through a glob pattern.
     */
   def getFiles(dir: String, hadoopConf: Configuration, recursive: Boolean = false): List[String] = {
-    getFiles(dir, FileSystem.get(hadoopConf), recursive)
+    getFiles(dir, new Path(dir).getFileSystem(hadoopConf), recursive)
   }
 
   /**

diff --git a/spark-cobol/src/main/scala/za/co/absa/cobrix/spark/cobol/utils/ResourceUtils.scala b/spark-cobol/src/main/scala/za/co/absa/cobrix/spark/cobol/utils/ResourceUtils.scala
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2018 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.cobrix.spark.cobol.utils
+
+import org.apache.commons.io.IOUtils
+
+object ResourceUtils {
+  /** Get resource file as a string. */
+  def readResourceAsString(path: String): String = IOUtils.toString(getClass.getResourceAsStream(path))
+}