Browse Source

Merge "Changes directory representation"

Jenkins 4 years ago
parent
commit
0e2de71ea3

+ 3
- 1
hadoop-swiftfs/src/main/java/org/apache/hadoop/fs/swift/http/SwiftRestClient.java View File

@@ -899,6 +899,7 @@ public final class SwiftRestClient {
899 899
    * Find objects in a directory
900 900
    *
901 901
    * @param path path prefix
902
+   * @param addTrailingSlash should a trailing slash be added if there isn't one
902 903
    * @param requestHeaders optional request headers
903 904
    * @return byte[] file data or null if the object was not found
904 905
    * @throws IOException on IO Faults
@@ -907,6 +908,7 @@ public final class SwiftRestClient {
907 908
    */
908 909
   public byte[] listDeepObjectsInDirectory(SwiftObjectPath path,
909 910
                                            boolean listDeep,
911
+                                           boolean addTrailingSlash,
910 912
                                        final Header... requestHeaders)
911 913
           throws IOException {
912 914
     preRemoteCommand("listDeepObjectsInDirectory");
@@ -918,7 +920,7 @@ public final class SwiftRestClient {
918 920
     if (object.startsWith("/")) {
919 921
       object = object.substring(1);
920 922
     }
921
-    if (!object.endsWith("/")) {
923
+    if (addTrailingSlash && !object.endsWith("/")) {
922 924
       object = object.concat("/");
923 925
     }
924 926
 

+ 23
- 13
hadoop-swiftfs/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftFileStatus.java View File

@@ -21,13 +21,16 @@ package org.apache.hadoop.fs.swift.snative;
21 21
 import org.apache.hadoop.fs.FileStatus;
22 22
 import org.apache.hadoop.fs.Path;
23 23
 import org.apache.hadoop.fs.permission.FsPermission;
24
+import org.apache.hadoop.fs.swift.util.SwiftObjectPath;
24 25
 
25 26
 /**
26 27
  * A subclass of {@link FileStatus} that contains the
27
- * Swift-specific rules of when a file is considered to be a directory.
28
+ * Swift-specific meta-data (e.g. DLO)
28 29
  */
29 30
 public class SwiftFileStatus extends FileStatus {
30 31
 
32
+  private SwiftObjectPath dloPrefix = null;
33
+
31 34
   public SwiftFileStatus() {
32 35
   }
33 36
 
@@ -35,7 +38,17 @@ public class SwiftFileStatus extends FileStatus {
35 38
                          boolean isdir,
36 39
                          int block_replication,
37 40
                          long blocksize, long modification_time, Path path) {
41
+    this(length, isdir, block_replication, blocksize, modification_time,
42
+            path, null);
43
+  }
44
+
45
+  public SwiftFileStatus(long length,
46
+                         boolean isdir,
47
+                         int block_replication,
48
+                         long blocksize, long modification_time, Path path,
49
+                         SwiftObjectPath dloPrefix) {
38 50
     super(length, isdir, block_replication, blocksize, modification_time, path);
51
+    this.dloPrefix = dloPrefix;
39 52
   }
40 53
 
41 54
   public SwiftFileStatus(long length,
@@ -50,17 +63,6 @@ public class SwiftFileStatus extends FileStatus {
50 63
             access_time, permission, owner, group, path);
51 64
   }
52 65
 
53
-  /**
54
-   * Declare that the path represents a directory, which in the
55
-   * SwiftNativeFileSystem means "is a directory or a 0 byte file"
56
-   *
57
-   * @return true if the status is considered to be a file
58
-   */
59
-  @Override
60
-  public boolean isDir() {
61
-    return super.isDir() || getLen() == 0;
62
-  }
63
-
64 66
   /**
65 67
    * A entry is a file if it is not a directory.
66 68
    * By implementing it <i>and not marking as an override</i> this
@@ -79,13 +81,21 @@ public class SwiftFileStatus extends FileStatus {
79 81
     return isDir();
80 82
   }
81 83
 
84
+  public boolean isDLO() {
85
+    return dloPrefix != null;
86
+  }
87
+
88
+  public SwiftObjectPath getDLOPrefix() {
89
+    return dloPrefix;
90
+  }
91
+
82 92
   @Override
83 93
   public String toString() {
84 94
     StringBuilder sb = new StringBuilder();
85 95
     sb.append(getClass().getSimpleName());
86 96
     sb.append("{ ");
87 97
     sb.append("path=").append(getPath());
88
-    sb.append("; isDirectory=").append(isDir());
98
+    sb.append("; isDirectory=").append(isDirectory());
89 99
     sb.append("; length=").append(getLen());
90 100
     sb.append("; blocksize=").append(getBlockSize());
91 101
     sb.append("; modification_time=").append(getModificationTime());

+ 21
- 27
hadoop-swiftfs/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystem.java View File

@@ -200,27 +200,6 @@ public class SwiftNativeFileSystem extends FileSystem {
200 200
     return store.getBlocksize();
201 201
   }
202 202
 
203
-  @Override
204
-  public boolean isFile(Path f) throws IOException {
205
-    try {
206
-      FileStatus fileStatus = getFileStatus(f);
207
-      return !SwiftUtils.isDirectory(fileStatus);
208
-    } catch (FileNotFoundException e) {
209
-      return false;               // f does not exist
210
-    }
211
-  }
212
-
213
-  @Override
214
-  public boolean isDirectory(Path f) throws IOException {
215
-
216
-    try {
217
-      FileStatus fileStatus = getFileStatus(f);
218
-      return SwiftUtils.isDirectory(fileStatus);
219
-    } catch (FileNotFoundException e) {
220
-      return false;               // f does not exist
221
-    }
222
-  }
223
-
224 203
   /**
225 204
    * Return an array containing hostnames, offset and size of
226 205
    * portions of the given file.  For a nonexistent
@@ -239,6 +218,9 @@ public class SwiftNativeFileSystem extends FileSystem {
239 218
     if (file == null) {
240 219
       return null;
241 220
     }
221
+    if (file.isDir()) {
222
+      return new BlockLocation[0];
223
+    }
242 224
 
243 225
     if (start < 0 || len < 0) {
244 226
       throw new IllegalArgumentException("Negative start or len parameter" +
@@ -251,11 +233,15 @@ public class SwiftNativeFileSystem extends FileSystem {
251 233
     // Check if requested file in Swift is more than 5Gb. In this case
252 234
     // each block has its own location -which may be determinable
253 235
     // from the Swift client API, depending on the remote server
254
-    final FileStatus[] listOfFileBlocks = store.listSubPaths(file.getPath(),
255
-                                                             false,
256
-                                                             true);
236
+    final FileStatus[] listOfFileBlocks;
237
+    if (file instanceof SwiftFileStatus && ((SwiftFileStatus)file).isDLO()) {
238
+      listOfFileBlocks = store.listSegments((SwiftFileStatus)file, true);
239
+    } else {
240
+      listOfFileBlocks = null;
241
+    }
242
+
257 243
     List<URI> locations = new ArrayList<URI>();
258
-    if (listOfFileBlocks.length > 1) {
244
+    if (listOfFileBlocks != null && listOfFileBlocks.length > 1) {
259 245
       for (FileStatus fileStatus : listOfFileBlocks) {
260 246
         if (SwiftObjectPath.fromPath(uri, fileStatus.getPath())
261 247
                 .equals(SwiftObjectPath.fromPath(uri, file.getPath()))) {
@@ -386,7 +372,7 @@ public class SwiftNativeFileSystem extends FileSystem {
386 372
       //find out about the path
387 373
       fileStatus = getFileStatus(directory);
388 374
 
389
-      if (!SwiftUtils.isDirectory(fileStatus)) {
375
+      if (!fileStatus.isDir()) {
390 376
         //if it's a file, raise an error
391 377
         throw new SwiftNotDirectoryException(directory,
392 378
                 String.format(": can't mkdir since it exists and is not a directory: %s",
@@ -433,7 +419,15 @@ public class SwiftNativeFileSystem extends FileSystem {
433 419
     if (LOG.isDebugEnabled()) {
434 420
       LOG.debug("SwiftFileSystem.listStatus for: " + path);
435 421
     }
436
-    return store.listSubPaths(makeAbsolute(path), false, true);
422
+    Path absolutePath = makeAbsolute(path);
423
+    try {
424
+      return store.listSubPaths(absolutePath, false, true);
425
+    } catch (FileNotFoundException e) {
426
+      /* path is not directory. try to get file status */
427
+      return new FileStatus[] {
428
+          getFileStatus(absolutePath)
429
+      };
430
+    }
437 431
   }
438 432
 
439 433
   /**

+ 177
- 57
hadoop-swiftfs/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystemStore.java View File

@@ -210,15 +210,38 @@ public class SwiftNativeFileSystemStore {
210 210
     throws IOException, FileNotFoundException {
211 211
 
212 212
     SwiftObjectPath objectPath = toObjectPath(path);
213
-    final Header[] headers = stat(objectPath, newest);
214
-    //no headers is treated as a missing file
215
-    if (headers.length == 0) {
213
+
214
+    // remove trailing slash because FileStatus must not include that
215
+    Path statusPath = path;
216
+    if (statusPath.toUri().toString().endsWith("/")) {
217
+      String pathUri = statusPath.toUri().toString();
218
+      if (pathUri.length() > 1)
219
+        statusPath = new Path(pathUri.substring(0, pathUri.length() - 1));
220
+    }
221
+
222
+    Header[] headers = null;
223
+    try {
224
+      headers = stat(objectPath, newest);
225
+    } catch (FileNotFoundException e) {
226
+      // if path is pseudo-directory, ignore FileNotFoundException.
227
+    }
228
+    //no headers is treated as a missing file or pseudo-directory
229
+    if (headers == null || headers.length == 0) {
230
+      if (existsPseudoDirectory(objectPath)) {
231
+        return new SwiftFileStatus(0,
232
+                                   true,
233
+                                   1,
234
+                                   getBlocksize(),
235
+                                   System.currentTimeMillis(),
236
+                                   getCorrectSwiftPath(statusPath));
237
+      }
216 238
       throw new FileNotFoundException("Not Found " + path.toUri());
217 239
     }
218 240
 
219 241
     boolean isDir = false;
220 242
     long length = 0;
221 243
     long lastModified = 0 ;
244
+    SwiftObjectPath dloPrefix = null;
222 245
     for (Header header : headers) {
223 246
       String headerName = header.getName();
224 247
       if (headerName.equals(SwiftProtocolConstants.X_CONTAINER_OBJECT_COUNT) ||
@@ -237,18 +260,28 @@ public class SwiftNativeFileSystemStore {
237 260
           throw new SwiftException("Failed to parse " + header.toString(), e);
238 261
         }
239 262
       }
263
+      if (headerName.equals(SwiftProtocolConstants.X_OBJECT_MANIFEST)) {
264
+        String[] values = header.getValue().split("/", 2);
265
+        if (values.length == 2) {
266
+          dloPrefix = new SwiftObjectPath(values[0], "/" + values[1]);
267
+        }
268
+      }
240 269
     }
241 270
     if (lastModified == 0) {
242 271
       lastModified = System.currentTimeMillis();
243 272
     }
273
+    if (objectPath.toString().endsWith("/")) {
274
+      isDir = true;
275
+    }
244 276
 
245
-    Path correctSwiftPath = getCorrectSwiftPath(path);
277
+    Path correctSwiftPath = getCorrectSwiftPath(statusPath);
246 278
     return new SwiftFileStatus(length,
247 279
                                isDir,
248 280
                                1,
249 281
                                getBlocksize(),
250 282
                                lastModified,
251
-                               correctSwiftPath);
283
+                               correctSwiftPath,
284
+                               dloPrefix);
252 285
   }
253 286
 
254 287
   private Header[] stat(SwiftObjectPath objectPath, boolean newest) throws
@@ -264,6 +297,39 @@ public class SwiftNativeFileSystemStore {
264 297
     return headers;
265 298
   }
266 299
 
300
+  private boolean existsPseudoDirectory(SwiftObjectPath path) {
301
+    try {
302
+      String pseudoDirName = path.getObject();
303
+      if (pseudoDirName.endsWith("/")) {
304
+        String obj = path.getObject();
305
+        path = new SwiftObjectPath(path.getContainer(),
306
+                                   obj.substring(0, obj.length() - 1));
307
+      } else {
308
+        pseudoDirName = pseudoDirName.concat("/");
309
+      }
310
+
311
+      final byte[] bytes;
312
+      bytes = swiftRestClient.listDeepObjectsInDirectory(path, false, false);
313
+
314
+      final CollectionType collectionType = JSONUtil.getJsonMapper().
315
+        getTypeFactory().constructCollectionType(List.class,
316
+                                                 SwiftObjectFileStatus.class);
317
+
318
+      final List<SwiftObjectFileStatus> fileStatusList =
319
+        JSONUtil.toObject(new String(bytes), collectionType);
320
+
321
+      for (SwiftObjectFileStatus status : fileStatusList) {
322
+        if (pseudoDirName.equals(status.getSubdir())) {
323
+          return true;
324
+        }
325
+      }
326
+      return false;
327
+
328
+    } catch (Exception e) {
329
+      return false;
330
+    }
331
+  }
332
+
267 333
   /**
268 334
    * Get the object as an input stream
269 335
    *
@@ -401,18 +467,22 @@ public class SwiftNativeFileSystemStore {
401 467
    * @param path working path
402 468
    * @param listDeep ask for all the data
403 469
    * @param newest ask for the newest data
470
+   * @param addTrailingSlash should a trailing slash be added if there isn't one
404 471
    * @return Collection of file statuses
405 472
    * @throws IOException IO problems
406 473
    * @throws FileNotFoundException if the path does not exist
407 474
    */
408 475
   private List<FileStatus> listDirectory(SwiftObjectPath path,
409 476
                                          boolean listDeep,
410
-                                         boolean newest) throws IOException {
477
+                                         boolean newest,
478
+                                         boolean addTrailingSlash)
479
+      throws IOException {
411 480
     final byte[] bytes;
412 481
     final ArrayList<FileStatus> files = new ArrayList<FileStatus>();
413 482
     final Path correctSwiftPath = getCorrectSwiftPath(path);
414 483
     try {
415
-      bytes = swiftRestClient.listDeepObjectsInDirectory(path, listDeep);
484
+      bytes = swiftRestClient.listDeepObjectsInDirectory(path, listDeep,
485
+                                                         addTrailingSlash);
416 486
     } catch (FileNotFoundException e) {
417 487
       if (LOG.isDebugEnabled()) {
418 488
         LOG.debug("" +
@@ -468,14 +538,25 @@ public class SwiftNativeFileSystemStore {
468 538
       return files;
469 539
     }
470 540
 
541
+    String pathWithSlash = path.getObject();
542
+    if (!pathWithSlash.endsWith("/")) {
543
+      pathWithSlash = pathWithSlash.concat("/");
544
+    }
471 545
     for (SwiftObjectFileStatus status : fileStatusList) {
472
-      if (status.getName() != null) {
473
-          files.add(new SwiftFileStatus(status.getBytes(),
474
-                  status.getBytes() == 0,
475
-                  1,
476
-                  getBlocksize(),
477
-                  status.getLast_modified().getTime(),
478
-                  getCorrectSwiftPath(new Path(status.getName()))));
546
+      String name = status.getName();
547
+      if (name == null) {
548
+        name = status.getSubdir();
549
+      }
550
+      if (name == null || name.equals(pathWithSlash)) {
551
+        continue;
552
+      }
553
+
554
+      if (!name.endsWith("/")) {
555
+        final Path filePath = getCorrectSwiftPath(new Path(name));
556
+        files.add(getObjectMetadata(filePath, newest));
557
+      } else {
558
+        final Path dirPath = getCorrectSwiftPath(toDirPath(new Path(name)));
559
+        files.add(getObjectMetadata(dirPath, newest));
479 560
       }
480 561
     }
481 562
 
@@ -498,7 +579,7 @@ public class SwiftNativeFileSystemStore {
498 579
                                    boolean recursive,
499 580
                                    boolean newest) throws IOException {
500 581
     final Collection<FileStatus> fileStatuses;
501
-    fileStatuses = listDirectory(toDirPath(path), recursive, newest);
582
+    fileStatuses = listDirectory(toDirPath(path), recursive, newest, true);
502 583
     return fileStatuses.toArray(new FileStatus[fileStatuses.size()]);
503 584
   }
504 585
 
@@ -527,7 +608,7 @@ public class SwiftNativeFileSystemStore {
527 608
 
528 609
   private SwiftObjectPath toDirPath(Path path) throws
529 610
           SwiftConfigurationException {
530
-    return SwiftObjectPath.fromPath(uri, path, false);
611
+    return SwiftObjectPath.fromPath(uri, path, true);
531 612
   }
532 613
 
533 614
   private SwiftObjectPath toObjectPath(Path path) throws
@@ -554,12 +635,17 @@ public class SwiftNativeFileSystemStore {
554 635
   /**
555 636
    * deletes object from Swift
556 637
    *
557
-   * @param path path to delete
638
+   * @param status FileStatus to delete
558 639
    * @return true if the path was deleted by this specific operation.
559 640
    * @throws IOException on a failure
560 641
    */
561
-  public boolean deleteObject(Path path) throws IOException {
562
-    SwiftObjectPath swiftObjectPath = toObjectPath(path);
642
+  public boolean deleteObject(FileStatus status) throws IOException {
643
+    SwiftObjectPath swiftObjectPath;
644
+    if (status.isDir()) {
645
+      swiftObjectPath = toDirPath(status.getPath());
646
+    } else {
647
+      swiftObjectPath = toObjectPath(status.getPath());
648
+    }
563 649
     if (!SwiftUtils.isRootDir(swiftObjectPath)) {
564 650
       return swiftRestClient.delete(swiftObjectPath);
565 651
     } else {
@@ -570,18 +656,6 @@ public class SwiftNativeFileSystemStore {
570 656
     }
571 657
   }
572 658
 
573
-  /**
574
-   * deletes a directory from Swift. This is not recursive
575
-   *
576
-   * @param path path to delete
577
-   * @return true if the path was deleted by this specific operation -or
578
-   *         the path was root and not acted on.
579
-   * @throws IOException on a failure
580
-   */
581
-  public boolean rmdir(Path path) throws IOException {
582
-    return deleteObject(path);
583
-  }
584
-
585 659
   /**
586 660
    * Does the object exist
587 661
    *
@@ -635,22 +709,32 @@ public class SwiftNativeFileSystemStore {
635 709
     }
636 710
     boolean renamingOnToSelf = src.equals(dst);
637 711
 
638
-    SwiftObjectPath srcObject = toObjectPath(src);
639
-    SwiftObjectPath destObject = toObjectPath(dst);
640
-
712
+    final SwiftFileStatus srcMetadata;
713
+    srcMetadata = getObjectMetadata(src);
714
+    SwiftObjectPath srcObject;
715
+    if (srcMetadata.isDirectory()) {
716
+      srcObject = toDirPath(src);
717
+    } else {
718
+      srcObject = toObjectPath(src);
719
+    }
641 720
     if (SwiftUtils.isRootDir(srcObject)) {
642 721
       throw new SwiftOperationFailedException("cannot rename root dir");
643 722
     }
644 723
 
645
-    final SwiftFileStatus srcMetadata;
646
-    srcMetadata = getObjectMetadata(src);
647 724
     SwiftFileStatus dstMetadata;
725
+    SwiftObjectPath destObject;
648 726
     try {
649 727
       dstMetadata = getObjectMetadata(dst);
728
+      if (dstMetadata.isDirectory()) {
729
+        destObject = toDirPath(dst);
730
+      } else {
731
+        destObject = toObjectPath(dst);
732
+      }
650 733
     } catch (FileNotFoundException e) {
651 734
       //destination does not exist.
652 735
       LOG.debug("Destination does not exist");
653 736
       dstMetadata = null;
737
+      destObject = toObjectPath(dst);
654 738
     }
655 739
 
656 740
     //check to see if the destination parent directory exists
@@ -670,13 +754,13 @@ public class SwiftNativeFileSystemStore {
670 754
     }
671 755
 
672 756
     boolean destExists = dstMetadata != null;
673
-    boolean destIsDir = destExists && SwiftUtils.isDirectory(dstMetadata);
757
+    boolean destIsDir = destExists && dstMetadata.isDirectory();
674 758
     //calculate the destination
675 759
     SwiftObjectPath destPath;
676 760
 
677 761
     //enum the child entries and everything underneath
678
-    List<FileStatus> childStats = listDirectory(srcObject, true, true);
679
-    boolean srcIsFile = !srcMetadata.isDir();
762
+    List<FileStatus> childStats = listDirectory(srcObject, true, true, true);
763
+    boolean srcIsFile = !srcMetadata.isDirectory();
680 764
     if (srcIsFile) {
681 765
 
682 766
       //source is a simple file OR a partitioned file
@@ -718,7 +802,7 @@ public class SwiftNativeFileSystemStore {
718 802
         copyObject(srcObject, destPath);
719 803
         for (FileStatus stat : childStats) {
720 804
           SwiftUtils.debug(LOG, "Deleting partitioned file %s ", stat);
721
-          deleteObject(stat.getPath());
805
+          deleteObject(stat);
722 806
         }
723 807
 
724 808
         swiftRestClient.delete(srcObject);
@@ -746,7 +830,7 @@ public class SwiftNativeFileSystemStore {
746 830
         // #3 destination doesn't exist: create a new dir with that name
747 831
         targetPath = dst;
748 832
       }
749
-      SwiftObjectPath targetObjectPath = toObjectPath(targetPath);
833
+      SwiftObjectPath targetObjectPath = toDirPath(targetPath);
750 834
       //final check for any recursive operations
751 835
       if (srcObject.isEqualToOrParentOf(targetObjectPath)) {
752 836
         //you can't rename a directory onto itself
@@ -780,11 +864,19 @@ public class SwiftNativeFileSystemStore {
780 864
                   + "; copyDestSubPath=" + copyDestSubPath
781 865
                   + "; copyDestPath=" + copyDestPath);
782 866
         }
783
-        SwiftObjectPath copyDestination = toObjectPath(copyDestPath);
867
+        SwiftObjectPath copySource;
868
+        SwiftObjectPath copyDestination;
869
+
870
+        if (fileStatus.isDir()) {
871
+          copySource = toDirPath(copySourcePath);
872
+          copyDestination = toDirPath(copyDestPath);
873
+        } else {
874
+          copySource = toObjectPath(copySourcePath);
875
+          copyDestination = toObjectPath(copyDestPath);
876
+        }
784 877
 
785 878
         try {
786
-          copyThenDeleteObject(toObjectPath(copySourcePath),
787
-                  copyDestination);
879
+          copyThenDeleteObject(copySource, copyDestination);
788 880
         } catch (FileNotFoundException e) {
789 881
           LOG.info("Skipping rename of " + copySourcePath);
790 882
         }
@@ -1008,14 +1100,12 @@ public class SwiftNativeFileSystemStore {
1008 1100
     //don't mind if the directory has changed
1009 1101
     //list all entries under this directory.
1010 1102
     //this will throw FileNotFoundException if the file isn't there
1011
-    FileStatus[] statuses = listSubPaths(absolutePath, true, askForNewest);
1012
-    if (statuses == null) {
1013
-      //the directory went away during the non-atomic stages of the operation.
1014
-      // Return false as it was not this thread doing the deletion.
1015
-      SwiftUtils.debug(LOG, "Path '%s' has no status -it has 'gone away'",
1016
-                       absolutePath,
1017
-                       recursive);
1018
-      return false;
1103
+    FileStatus[] statuses;
1104
+    try {
1105
+      statuses = listSubPaths(absolutePath, true, askForNewest);
1106
+    } catch (IOException e) {
1107
+      // absolutePath is nonexistent
1108
+      statuses = new FileStatus[0];
1019 1109
     }
1020 1110
     int filecount = statuses.length;
1021 1111
     SwiftUtils.debug(LOG, "Path '%s' %d status entries'",
@@ -1024,7 +1114,7 @@ public class SwiftNativeFileSystemStore {
1024 1114
 
1025 1115
     if (filecount == 0) {
1026 1116
       //it's an empty directory or a path
1027
-      rmdir(absolutePath);
1117
+      deleteObject(fileStatus);
1028 1118
       return true;
1029 1119
     }
1030 1120
 
@@ -1036,14 +1126,14 @@ public class SwiftNativeFileSystemStore {
1036 1126
       // 1 entry => simple file and it is the target
1037 1127
       //simple file: delete it
1038 1128
       SwiftUtils.debug(LOG, "Deleting simple file %s", absolutePath);
1039
-      deleteObject(absolutePath);
1129
+      deleteObject(fileStatus);
1040 1130
       return true;
1041 1131
     }
1042 1132
 
1043 1133
     //>1 entry implies directory with children. Run through them,
1044 1134
     // but first check for the recursive flag and reject it *unless it looks
1045 1135
     // like a partitioned file (len > 0 && has children)
1046
-    if (!fileStatus.isDir()) {
1136
+    if (!fileStatus.isDirectory()) {
1047 1137
       LOG.debug("Multiple child entries but entry has data: assume partitioned");
1048 1138
     } else if (!recursive) {
1049 1139
       //if there are children, unless this is a recursive operation, fail immediately
@@ -1057,7 +1147,7 @@ public class SwiftNativeFileSystemStore {
1057 1147
     for (FileStatus entryStatus : statuses) {
1058 1148
       Path entryPath = entryStatus.getPath();
1059 1149
       try {
1060
-        boolean deleted = deleteObject(entryPath);
1150
+        boolean deleted = deleteObject(entryStatus);
1061 1151
         if (!deleted) {
1062 1152
           SwiftUtils.debug(LOG, "Failed to delete entry '%s'; continuing",
1063 1153
                            entryPath);
@@ -1072,8 +1162,38 @@ public class SwiftNativeFileSystemStore {
1072 1162
     }
1073 1163
     //now delete self
1074 1164
     SwiftUtils.debug(LOG, "Deleting base entry %s", absolutePath);
1075
-    deleteObject(absolutePath);
1165
+    deleteObject(fileStatus);
1076 1166
 
1077 1167
     return true;
1078 1168
   }
1169
+
1170
+  /**
1171
+   * List all segments in dynamic large object.
1172
+   *
1173
+   * @param file   SwiftFileStatus of large object
1174
+   * @param newest ask for the newest, or can some out of date data work?
1175
+   * @return the file statuses, or an empty array if there are no segments
1176
+   * @throws IOException           on IO problems
1177
+   */
1178
+  public FileStatus[] listSegments(SwiftFileStatus file, boolean newest)
1179
+      throws IOException {
1180
+
1181
+    if (file.getDLOPrefix() == null) {
1182
+      return new FileStatus[0];
1183
+    }
1184
+
1185
+    final List<FileStatus> objects;
1186
+    objects = listDirectory(file.getDLOPrefix(), true, newest, false);
1187
+
1188
+    final ArrayList<FileStatus> segments;
1189
+    segments = new ArrayList<FileStatus>(objects.size());
1190
+
1191
+    for (FileStatus status : objects) {
1192
+      if (!status.isDir()) {
1193
+        segments.add(status);
1194
+      }
1195
+    }
1196
+
1197
+    return segments.toArray(new FileStatus[segments.size()]);
1198
+  }
1079 1199
 }

+ 1
- 1
hadoop-swiftfs/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftObjectFileStatus.java View File

@@ -26,7 +26,7 @@ import java.util.Date;
26 26
  * DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
27 27
  */
28 28
 
29
-class SwiftObjectFileStatus {
29
+public class SwiftObjectFileStatus {
30 30
   private long bytes;
31 31
   private String content_type;
32 32
   private String hash;

+ 0
- 23
hadoop-swiftfs/src/main/java/org/apache/hadoop/fs/swift/util/SwiftUtils.java View File

@@ -58,29 +58,6 @@ public final class SwiftUtils {
58 58
     return result.toString();
59 59
   }
60 60
 
61
-  /**
62
-   * This test contains the is-directory logic for Swift, so if
63
-   * changed there is only one place for it.
64
-   *
65
-   * @param fileStatus status to examine
66
-   * @return true if we consider this status to be representative of a
67
-   *         directory.
68
-   */
69
-  public static boolean isDirectory(FileStatus fileStatus) {
70
-    return fileStatus.isDir() || isFilePretendingToBeDirectory(fileStatus);
71
-  }
72
-
73
-  /**
74
-   * Test for the entry being a file that is treated as if it is a
75
-   * directory
76
-   *
77
-   * @param fileStatus status
78
-   * @return true if it meets the rules for being a directory
79
-   */
80
-  public static boolean isFilePretendingToBeDirectory(FileStatus fileStatus) {
81
-    return fileStatus.getLen() == 0;
82
-  }
83
-
84 61
   /**
85 62
    * Predicate: Is a swift object referring to the root direcory?
86 63
    * @param swiftObject object to probe

+ 6
- 0
hadoop-swiftfs/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemBasicOps.java View File

@@ -286,4 +286,10 @@ public class TestSwiftFileSystemBasicOps extends SwiftFileSystemBaseTest {
286 286
     }
287 287
   }
288 288
 
289
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
290
+  public void testExistsRoot() throws Exception {
291
+    Path path = new Path("/");
292
+    assertTrue("exists('/') returned false", fs.exists(path));
293
+  }
294
+
289 295
 }

+ 56
- 5
hadoop-swiftfs/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemDirectories.java View File

@@ -20,11 +20,21 @@ package org.apache.hadoop.fs.swift;
20 20
 
21 21
 import org.apache.hadoop.fs.FileStatus;
22 22
 import org.apache.hadoop.fs.Path;
23
+import org.apache.hadoop.fs.swift.http.SwiftRestClient;
23 24
 import org.apache.hadoop.fs.swift.snative.SwiftFileStatus;
25
+import org.apache.hadoop.fs.swift.snative.SwiftObjectFileStatus;
26
+import org.apache.hadoop.fs.swift.util.JSONUtil;
27
+import org.apache.hadoop.fs.swift.util.SwiftObjectPath;
24 28
 import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
29
+import org.codehaus.jackson.map.type.CollectionType;
25 30
 import org.junit.Test;
26 31
 
27 32
 import java.io.FileNotFoundException;
33
+import java.net.URI;
34
+import java.util.ArrayList;
35
+import java.util.List;
36
+
37
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.cleanup;
28 38
 
29 39
 /**
30 40
  * Test swift-specific directory logic.
@@ -35,16 +45,16 @@ public class TestSwiftFileSystemDirectories extends SwiftFileSystemBaseTest {
35 45
 
36 46
   /**
37 47
    * Asserts that a zero byte file has a status of file and not
38
-   * directory or symlink
48
+   * file or symlink
39 49
    *
40 50
    * @throws Exception on failures
41 51
    */
42 52
   @Test(timeout = SWIFT_TEST_TIMEOUT)
43
-  public void testZeroByteFilesAreDirectories() throws Exception {
53
+  public void testZeroByteFilesAreFiles() throws Exception {
44 54
     Path src = path("/test/testZeroByteFilesAreFiles");
45 55
     //create a zero byte file
46 56
     SwiftTestUtils.touch(fs, src);
47
-    SwiftTestUtils.assertIsDirectory(fs, src);
57
+    SwiftTestUtils.assertIsFile(fs, src);
48 58
   }
49 59
 
50 60
   @Test(timeout = SWIFT_TEST_TIMEOUT)
@@ -79,8 +89,8 @@ public class TestSwiftFileSystemDirectories extends SwiftFileSystemBaseTest {
79 89
 
80 90
     Path src = path("/test/file");
81 91
 
82
-    //create a zero byte file
83
-    SwiftTestUtils.touch(fs, src);
92
+    //create a directory
93
+    fs.mkdirs(src);
84 94
     //stat it
85 95
     statuses = fs.listStatus(test);
86 96
     statusString = statusToString(test.toString(), statuses);
@@ -138,4 +148,45 @@ public class TestSwiftFileSystemDirectories extends SwiftFileSystemBaseTest {
138 148
     assertFalse(status.isDir());
139 149
   }
140 150
 
151
+  /**
152
+   * Asserts that a mkdir with trailing slash makes single object only
153
+   *
154
+   * @throws Exception on failures
155
+   */
156
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
157
+  public void testDirectoriesUseURI() throws Exception {
158
+    cleanup("testDirectoriesUseURI", fs, "/");
159
+    Path test = new Path(fs.getUri().resolve("/test/"));
160
+    mkdirs(test);
161
+    assertExists("created test directory", test);
162
+    FileStatus[] statuses = fs.listStatus(test);
163
+    String statusString = statusToString(test.toString(), statuses);
164
+    assertEquals("Wrong number of elements in file status " + statusString, 0,
165
+                 statuses.length);
166
+
167
+    String[] objects = getRawObjectNames();
168
+    assertEquals("Wrong number of objects in swift", 1, objects.length);
169
+    assertEquals("Wrong directory name", "/test/", objects[0]);
170
+  }
171
+
172
+  private String[] getRawObjectNames() throws Exception {
173
+    SwiftRestClient client;
174
+    client = SwiftRestClient.getInstance(fs.getUri(), fs.getConf());
175
+    SwiftObjectPath path = SwiftObjectPath.fromPath(fs.getUri(), new Path("/"));
176
+    byte[] bytes = client.listDeepObjectsInDirectory(path, true, true);
177
+    final CollectionType collectionType = JSONUtil.getJsonMapper().
178
+      getTypeFactory().constructCollectionType(List.class,
179
+                                               SwiftObjectFileStatus.class);
180
+    final List<SwiftObjectFileStatus> fileStatusList =
181
+      JSONUtil.toObject(new String(bytes), collectionType);
182
+    final ArrayList<String> objects = new ArrayList();
183
+    for (SwiftObjectFileStatus status : fileStatusList) {
184
+      if (status.getName() != null) {
185
+        objects.add(status.getName());
186
+      } else if (status.getSubdir() != null) {
187
+        objects.add(status.getSubdir());
188
+      }
189
+    }
190
+    return objects.toArray(new String[objects.size()]);
191
+  }
141 192
 }

Loading…
Cancel
Save