Hiding a pseudo-directory that contains DLO segments

In SwiftNativeFileSystem implementation,
writes DLO manifest and segments to same location.

    largefile         <- manifest file
    largefile/000001  <- 1st segment
    largefile/000002  <- 2nd segment

And querying Swift to list objects (delimiter=/),
returns two object that is same name (sorted by name).

    largefile    <- manifest file
    largefile/   <- pseudo-directory

listStatus method returns two objects at above.
But FileSystem spec cannot exists file and directory in same name.

This patch modified listStatus's behavior to fix that problems
1. Ignores pseudo-directory entry If same named objects found
2. When called listStatus with manifest file path, returns a manifest file
   instead of the segments. (currently, returns list of segments)

Change-Id: Ic8835ba170c52ebac405315b44d59b4935909d51
Closes-Bug: 1436278

hadoop-swiftfs/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystem.java

@@ -235,7 +235,7 @@ public class SwiftNativeFileSystem extends FileSystem {
     // from the Swift client API, depending on the remote server
     final FileStatus[] listOfFileBlocks;
     if (file instanceof SwiftFileStatus && ((SwiftFileStatus)file).isDLO()) {
-      listOfFileBlocks = store.listSegments((SwiftFileStatus)file, true);
+      listOfFileBlocks = store.listSegments(file, true);
     } else {
       listOfFileBlocks = null;
     }
@@ -420,13 +420,11 @@ public class SwiftNativeFileSystem extends FileSystem {
       LOG.debug("SwiftFileSystem.listStatus for: " + path);
     }
     Path absolutePath = makeAbsolute(path);
-    try {
+    FileStatus status = getFileStatus(absolutePath);
+    if (status.isDir()) {
       return store.listSubPaths(absolutePath, false, true);
-    } catch (FileNotFoundException e) {
-      /* path is not directory. try to get file status */
-      return new FileStatus[] {
-          getFileStatus(absolutePath)
-      };
+    } else {
+      return new FileStatus[] {status};
     }
   }
 

hadoop-swiftfs/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystemStore.java

@@ -544,6 +544,7 @@ public class SwiftNativeFileSystemStore {
     if (!pathWithSlash.endsWith("/")) {
       pathWithSlash = pathWithSlash.concat("/");
     }
+    String prevObjName = "";
     for (SwiftObjectFileStatus status : fileStatusList) {
       String name = status.getName();
       if (name == null) {
@@ -556,7 +557,13 @@ public class SwiftNativeFileSystemStore {
       if (!name.endsWith("/")) {
         final Path filePath = getCorrectSwiftPath(new Path(name));
         files.add(getObjectMetadata(filePath, newest));
+        prevObjName = name;
       } else {
+        if (prevObjName.length() + 1 == name.length() &&
+            name.startsWith(prevObjName)) {
+          // Ignore because this directory object is stored DLO segment file
+          continue;
+        }
         final Path dirPath = getCorrectSwiftPath(toDirPath(new Path(name)));
         files.add(getObjectMetadata(dirPath, newest));
       }
@@ -1195,15 +1202,15 @@ public class SwiftNativeFileSystemStore {
    * @return the file statuses, or an empty array if there are no segments
    * @throws IOException           on IO problems
    */
-  public FileStatus[] listSegments(SwiftFileStatus file, boolean newest)
+  public FileStatus[] listSegments(FileStatus file, boolean newest)
       throws IOException {
-
-    if (file.getDLOPrefix() == null) {
+    SwiftObjectPath prefix = ((SwiftFileStatus)file).getDLOPrefix();
+    if (prefix == null) {
       return new FileStatus[0];
     }
 
     final List<FileStatus> objects;
-    objects = listDirectory(file.getDLOPrefix(), true, newest, false);
+    objects = listDirectory(prefix, true, newest, false);
 
     final ArrayList<FileStatus> segments;
     segments = new ArrayList<FileStatus>(objects.size());

hadoop-swiftfs/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemPartitionedUploads.java

@@ -321,7 +321,7 @@ public class TestSwiftFileSystemPartitionedUploads extends
     //compare data
     SwiftTestUtils.compareByteArrays(src, dest, len);
     //finally, check the data
-    FileStatus[] stats = fs.listStatus(path);
+    FileStatus[] stats = getStore().listSegments(fs.getFileStatus(path), true);
     assertEquals("wrong entry count in "
                  + SwiftTestUtils.dumpStats(path.toString(), stats),
                  expected, stats.length);
@@ -470,9 +470,24 @@ public class TestSwiftFileSystemPartitionedUploads extends
     //compare data
     SwiftTestUtils.compareByteArrays(src, dest, len);
     //finally, check the data
-    FileStatus[] stats = fs.listStatus(path);
+    FileStatus[] stats = getStore().listSegments(fs.getFileStatus(path), true);
     assertEquals("wrong entry count in "
                  + SwiftTestUtils.dumpStats(path.toString(), stats),
                  expected, stats.length);
   }
+
+  /**
+   * Tests directory visibility that stored partitioned file
+   * @throws Throwable
+   */
+  @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
+  public void testDirectoryVisibility() throws Throwable {
+    final Path filePath = path("/test/file");
+    final byte[] src = SwiftTestUtils.dataset(PART_SIZE_BYTES * 4, 32, 144);
+    FSDataOutputStream out = fs.create(filePath, false);
+    out.write(src, 0, src.length);
+    out.close();
+    assertEquals(1, fs.listStatus(path("/test")).length);
+    assertEquals(1, fs.listStatus(filePath).length);
+  }
 }