401 lines
12 KiB
Java
401 lines
12 KiB
Java
/*
|
|
* Licensed to the Apache Software Foundation (ASF) under one
|
|
* or more contributor license agreements. See the NOTICE file
|
|
* distributed with this work for additional information
|
|
* regarding copyright ownership. The ASF licenses this file
|
|
* to you under the Apache License, Version 2.0 (the
|
|
* "License"); you may not use this file except in compliance
|
|
* with the License. You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
package org.apache.hadoop.fs.swift;
|
|
|
|
import org.apache.commons.logging.Log;
|
|
import org.apache.commons.logging.LogFactory;
|
|
import org.apache.hadoop.conf.Configuration;
|
|
import org.apache.hadoop.fs.FSDataOutputStream;
|
|
import org.apache.hadoop.fs.FileStatus;
|
|
import org.apache.hadoop.fs.Path;
|
|
import org.apache.hadoop.fs.swift.exceptions.SwiftOperationFailedException;
|
|
import org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem;
|
|
import org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystemStore;
|
|
import org.apache.hadoop.fs.swift.util.DurationStats;
|
|
import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
|
|
import org.junit.After;
|
|
import org.junit.AfterClass;
|
|
import org.junit.Assert;
|
|
import org.junit.Assume;
|
|
import org.junit.Before;
|
|
|
|
import java.io.FileNotFoundException;
|
|
import java.io.IOException;
|
|
import java.io.OutputStream;
|
|
import java.net.URI;
|
|
import java.net.URISyntaxException;
|
|
import java.util.List;
|
|
|
|
import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.assertPathExists;
|
|
import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.cleanupInTeardown;
|
|
import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.getServiceURI;
|
|
import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.noteAction;
|
|
|
|
/**
|
|
* This is the base class for most of the Swift tests
|
|
*/
|
|
public class SwiftFileSystemBaseTest extends Assert implements
|
|
SwiftTestConstants {
|
|
|
|
protected static final Log LOG =
|
|
LogFactory.getLog(SwiftFileSystemBaseTest.class);
|
|
protected SwiftNativeFileSystem fs;
|
|
protected static SwiftNativeFileSystem lastFs;
|
|
protected byte[] data = SwiftTestUtils.dataset(getBlockSize() * 2, 0, 255);
|
|
private Configuration conf;
|
|
|
|
@Before
|
|
public void setUp() throws Exception {
|
|
noteAction("setup");
|
|
final URI uri = getFilesystemURI();
|
|
conf = createConfiguration();
|
|
|
|
fs = createSwiftFS();
|
|
try {
|
|
fs.initialize(uri, conf);
|
|
} catch (IOException e) {
|
|
//FS init failed, set it to null so that teardown doesn't
|
|
//attempt to use it
|
|
fs = null;
|
|
throw e;
|
|
}
|
|
//remember the last FS
|
|
lastFs = fs;
|
|
noteAction("setup complete");
|
|
}
|
|
|
|
/**
|
|
* Configuration generator. May be overridden to inject
|
|
* some custom options
|
|
* @return a configuration with which to create FS instances
|
|
*/
|
|
protected Configuration createConfiguration() {
|
|
return new Configuration();
|
|
}
|
|
|
|
@After
|
|
public void tearDown() throws Exception {
|
|
cleanupInTeardown(fs, "/test");
|
|
}
|
|
|
|
@AfterClass
|
|
public static void classTearDown() throws Exception {
|
|
if (lastFs != null) {
|
|
List<DurationStats> statistics = lastFs.getOperationStatistics();
|
|
for (DurationStats stat : statistics) {
|
|
LOG.info(stat.toString());
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Get the configuration used to set up the FS
|
|
* @return the configuration
|
|
*/
|
|
public Configuration getConf() {
|
|
return conf;
|
|
}
|
|
|
|
/**
|
|
* Describe the test, combining some logging with details
|
|
* for people reading the code
|
|
*
|
|
* @param description test description
|
|
*/
|
|
protected void describe(String description) {
|
|
noteAction(description);
|
|
}
|
|
|
|
protected URI getFilesystemURI() throws URISyntaxException, IOException {
|
|
return getServiceURI(createConfiguration());
|
|
}
|
|
|
|
protected SwiftNativeFileSystem createSwiftFS() throws IOException {
|
|
SwiftNativeFileSystem swiftNativeFileSystem =
|
|
new SwiftNativeFileSystem();
|
|
return swiftNativeFileSystem;
|
|
}
|
|
|
|
protected int getBlockSize() {
|
|
return 1024;
|
|
}
|
|
|
|
/**
|
|
* Is rename supported?
|
|
* @return true
|
|
*/
|
|
protected boolean renameSupported() {
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* assume in a test that rename is supported;
|
|
* skip it if not
|
|
*/
|
|
protected void assumeRenameSupported() {
|
|
Assume.assumeTrue(renameSupported());
|
|
}
|
|
|
|
/**
|
|
* Take an unqualified path, and qualify it w.r.t the
|
|
* current filesystem
|
|
* @param pathString source path
|
|
* @return a qualified path instance
|
|
*/
|
|
protected Path path(String pathString) {
|
|
return new Path(pathString).makeQualified(fs);
|
|
}
|
|
|
|
/**
|
|
* Get the filesystem
|
|
* @return the current FS
|
|
*/
|
|
public SwiftNativeFileSystem getFs() {
|
|
return fs;
|
|
}
|
|
|
|
/**
|
|
* Create a file using the standard {@link #data} bytes.
|
|
*
|
|
* @param path path to write
|
|
* @throws IOException on any problem
|
|
*/
|
|
protected void createFile(Path path) throws IOException {
|
|
createFile(path, data);
|
|
}
|
|
|
|
/**
|
|
* Create a file with the given data.
|
|
*
|
|
* @param path path to write
|
|
* @param sourceData source dataset
|
|
* @throws IOException on any problem
|
|
*/
|
|
protected void createFile(Path path, byte[] sourceData) throws IOException {
|
|
FSDataOutputStream out = fs.create(path);
|
|
out.write(sourceData, 0, sourceData.length);
|
|
out.close();
|
|
}
|
|
|
|
/**
|
|
* Create and then close a file
|
|
* @param path path to create
|
|
* @throws IOException on a failure
|
|
*/
|
|
protected void createEmptyFile(Path path) throws IOException {
|
|
FSDataOutputStream out = fs.create(path);
|
|
out.close();
|
|
}
|
|
|
|
/**
|
|
* Get the inner store -useful for lower level operations
|
|
*
|
|
* @return the store
|
|
*/
|
|
protected SwiftNativeFileSystemStore getStore() {
|
|
return fs.getStore();
|
|
}
|
|
|
|
/**
|
|
* Rename a path
|
|
* @param src source
|
|
* @param dst dest
|
|
* @param renameMustSucceed flag to say "this rename must exist"
|
|
* @param srcExists add assert that the source exists afterwards
|
|
* @param dstExists add assert the dest exists afterwards
|
|
* @throws IOException IO trouble
|
|
*/
|
|
protected void rename(Path src, Path dst, boolean renameMustSucceed,
|
|
boolean srcExists, boolean dstExists) throws IOException {
|
|
if (renameMustSucceed) {
|
|
renameToSuccess(src, dst, srcExists, dstExists);
|
|
} else {
|
|
renameToFailure(src, dst);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Get a string describing the outcome of a rename, by listing the dest
|
|
* path and its parent along with some covering text
|
|
* @param src source patj
|
|
* @param dst dest path
|
|
* @return a string for logs and exceptions
|
|
* @throws IOException IO problems
|
|
*/
|
|
private String getRenameOutcome(Path src, Path dst) throws IOException {
|
|
String lsDst = ls(dst);
|
|
Path parent = dst.getParent();
|
|
String lsParent = parent != null ? ls(parent) : "";
|
|
return " result of " + src + " => " + dst
|
|
+ " - " + lsDst
|
|
+ " \n" + lsParent;
|
|
}
|
|
|
|
/**
|
|
* Rename, expecting an exception to be thrown
|
|
*
|
|
* @param src source
|
|
* @param dst dest
|
|
* @throws IOException a failure other than an
|
|
* expected SwiftRenameException or FileNotFoundException
|
|
*/
|
|
protected void renameToFailure(Path src, Path dst) throws IOException {
|
|
try {
|
|
getStore().rename(src, dst);
|
|
fail("Expected failure renaming " + src + " to " + dst
|
|
+ "- but got success");
|
|
} catch (SwiftOperationFailedException e) {
|
|
LOG.debug("Rename failed (expected):" + e);
|
|
} catch (FileNotFoundException e) {
|
|
LOG.debug("Rename failed (expected):" + e);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Rename to success
|
|
*
|
|
* @param src source
|
|
* @param dst dest
|
|
* @param srcExists add assert that the source exists afterwards
|
|
* @param dstExists add assert the dest exists afterwards
|
|
* @throws SwiftOperationFailedException operation failure
|
|
* @throws IOException IO problems
|
|
*/
|
|
protected void renameToSuccess(Path src, Path dst,
|
|
boolean srcExists, boolean dstExists)
|
|
throws SwiftOperationFailedException, IOException {
|
|
getStore().rename(src, dst);
|
|
String outcome = getRenameOutcome(src, dst);
|
|
assertEquals("Source " + src + "exists: " + outcome,
|
|
srcExists, fs.exists(src));
|
|
assertEquals("Destination " + dstExists + " exists" + outcome,
|
|
dstExists, fs.exists(dst));
|
|
}
|
|
|
|
/**
|
|
* List a path in the test FS
|
|
* @param path path to list
|
|
* @return the contents of the path/dir
|
|
* @throws IOException IO problems
|
|
*/
|
|
protected String ls(Path path) throws IOException {
|
|
return SwiftTestUtils.ls(fs, path);
|
|
}
|
|
|
|
/**
|
|
* assert that a path exists
|
|
* @param message message to use in an assertion
|
|
* @param path path to probe
|
|
* @throws IOException IO problems
|
|
*/
|
|
public void assertExists(String message, Path path) throws IOException {
|
|
assertPathExists(fs, message, path);
|
|
}
|
|
|
|
/**
|
|
* assert that a path does not
|
|
* @param message message to use in an assertion
|
|
* @param path path to probe
|
|
* @throws IOException IO problems
|
|
*/
|
|
public void assertPathDoesNotExist(String message, Path path) throws
|
|
IOException {
|
|
SwiftTestUtils.assertPathDoesNotExist(fs, message, path);
|
|
}
|
|
|
|
/**
|
|
* Assert that a file exists and whose {@link FileStatus} entry
|
|
* declares that this is a file and not a symlink or directory.
|
|
*
|
|
* @param filename name of the file
|
|
* @throws IOException IO problems during file operations
|
|
*/
|
|
protected void assertIsFile(Path filename) throws IOException {
|
|
SwiftTestUtils.assertIsFile(fs, filename);
|
|
}
|
|
|
|
/**
|
|
* Assert that a file exists and whose {@link FileStatus} entry
|
|
* declares that this is a file and not a symlink or directory.
|
|
*
|
|
* @throws IOException IO problems during file operations
|
|
*/
|
|
protected void mkdirs(Path path) throws IOException {
|
|
assertTrue("Failed to mkdir" + path, fs.mkdirs(path));
|
|
}
|
|
|
|
/**
|
|
* Assert that a delete succeeded
|
|
* @param path path to delete
|
|
* @param recursive recursive flag
|
|
* @throws IOException IO problems
|
|
*/
|
|
protected void assertDeleted(Path path, boolean recursive) throws IOException {
|
|
SwiftTestUtils.assertDeleted(fs, path, recursive);
|
|
}
|
|
|
|
/**
|
|
* Assert that a value is not equal to the expected value
|
|
* @param message message if the two values are equal
|
|
* @param expected expected value
|
|
* @param actual actual value
|
|
*/
|
|
protected void assertNotEqual(String message, int expected, int actual) {
|
|
assertTrue(message,
|
|
actual != expected);
|
|
}
|
|
|
|
/**
|
|
* Get the number of partitions written from the Swift Native FS APIs
|
|
* @param out output stream
|
|
* @return the number of partitioned files written by the stream
|
|
*/
|
|
protected int getPartitionsWritten(FSDataOutputStream out) {
|
|
return SwiftNativeFileSystem.getPartitionsWritten(out);
|
|
}
|
|
|
|
/**
|
|
* Assert that the no. of partitions written matches expectations
|
|
* @param action operation (for use in the assertions)
|
|
* @param out output stream
|
|
* @param expected expected no. of partitions
|
|
*/
|
|
protected void assertPartitionsWritten(String action, FSDataOutputStream out,
|
|
long expected) {
|
|
OutputStream nativeStream = out.getWrappedStream();
|
|
int written = getPartitionsWritten(out);
|
|
if(written !=expected) {
|
|
Assert.fail(action + ": " +
|
|
TestSwiftFileSystemPartitionedUploads.WRONG_PARTITION_COUNT
|
|
+ " + expected: " + expected + " actual: " + written
|
|
+ " -- " + nativeStream);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Assert that the result value == -1; which implies
|
|
* that a read was successful
|
|
* @param text text to include in a message (usually the operation)
|
|
* @param result read result to validate
|
|
*/
|
|
protected void assertMinusOne(String text, int result) {
|
|
assertEquals(text + " wrong read result " + result, -1, result);
|
|
}
|
|
}
|