diff --git a/doc/source/admin/inspection/copy-inspection-data-swift.rst b/doc/source/admin/inspection/copy-inspection-data-swift.rst new file mode 100644 index 0000000000..634b665535 --- /dev/null +++ b/doc/source/admin/inspection/copy-inspection-data-swift.rst @@ -0,0 +1,114 @@ +Copy inspection data between Swift buckets +========================================== + +This script assumes that you have S3 credentials for the buckets. +You only have to configure the 5 configuration parameters. + +.. code-block:: python + + import boto3 + import json + from botocore.exceptions import ClientError + + # Configure your S3 buckets and Ceph endpoint + SOURCE_BUCKET = '' + DEST_BUCKET = '' + + # Ceph S3 configuration + CEPH_ENDPOINT = '' + CEPH_ACCESS_KEY = '' + CEPH_SECRET_KEY = '' + + def get_s3_client(): + """Initialize and return S3 client for Ceph""" + session = boto3.Session( + aws_secret_access_key=CEPH_SECRET_KEY, + aws_access_key_id=CEPH_ACCESS_KEY) + return session.client( + 's3', + endpoint_url=CEPH_ENDPOINT) + + def list_files_to_process(s3_client, bucket): + """List all files in bucket that don't end with '-UNPROCESSED'""" + files = [] + try: + paginator = s3_client.get_paginator('list_objects_v2') + for page in paginator.paginate(Bucket=bucket): + if 'Contents' in page: + for obj in page['Contents']: + key = obj['Key'] + if not key.endswith('-UNPROCESSED'): + files.append(key) + except ClientError as e: + print(f"Error listing files: {e}") + raise + return files + + def load_json_from_s3(s3_client, bucket, key): + """Load and parse JSON file from S3""" + try: + response = s3_client.get_object(Bucket=bucket, Key=key) + content = response['Body'].read().decode('utf-8') + return json.loads(content) + except ClientError as e: + print(f"Error reading {key}: {e}") + raise + except json.JSONDecodeError as e: + print(f"Error parsing JSON from {key}: {e}") + raise + + def save_json_to_s3(s3_client, bucket, key, data): + """Save JSON data to S3""" + try: + s3_client.put_object( + Bucket=bucket, + Key=key, + Body=json.dumps(data, indent=2), + ContentType='application/json' + ) + print(f"Saved: {key}") + except ClientError as e: + print(f"Error saving {key}: {e}") + raise + + def process_files(): + """Main processing function""" + s3_client = get_s3_client() + print(f"Fetching files from {SOURCE_BUCKET}...") + files = list_files_to_process(s3_client, SOURCE_BUCKET) + print(f"Found {len(files)} files to process") + + # Process each file + for file_key in files: + print(f"\nProcessing: {file_key}") + + try: + # Load JSON data + data = load_json_from_s3(s3_client, SOURCE_BUCKET, file_key) + + # Split data + inventory = data.pop('inventory', None) + plugin = data + + # Check if inventory key existed + if inventory is None: + print(f"Warning: 'inventory' key not found in {file_key}") + + # Generate output filenames + inventory_key = f"{file_key}-inventory" + plugin_key = f"{file_key}-plugin" + + # Save split files + if inventory is not None: + save_json_to_s3(s3_client, DEST_BUCKET, inventory_key, inventory) + if plugin is not None: + save_json_to_s3(s3_client, DEST_BUCKET, plugin_key, plugin) + + except Exception as e: + print(f"Failed to process {file_key}: {e}") + continue + + print("\nProcessing complete!") + + if __name__ == "__main__": + process_files() diff --git a/doc/source/admin/inspection/migration.rst b/doc/source/admin/inspection/migration.rst index bc00313418..46ff62f492 100644 --- a/doc/source/admin/inspection/migration.rst +++ b/doc/source/admin/inspection/migration.rst @@ -135,7 +135,8 @@ Other concerns ~~~~~~~~~~~~~~ * There is no way to migrate the inspection data automatically. You need to - repeat inspections or copy the data over manually. + repeat inspections or copy the data over manually. For migration of data + between Swift buckets, you could adapt :doc:`copy-inspection-data-swift`. Migration process -----------------