🧑💻Programming
How to programmatically work with AWS S3
Commands
List Buckets
aws s3 ls
import boto3
# Create a session using your AWS credentials
session = boto3.Session(
aws_access_key_id='your_access_key_id',
aws_secret_access_key='your_secret_access_key',
region_name='your_preferred_region'
)
# Create an S3 client
s3 = session.client('s3')
# List all buckets
response = s3.list_buckets()
# Print bucket names
print("Existing buckets:")
for bucket in response['Buckets']:
print(f" {bucket['Name']}")
Copy Bucket File(s) Locally
aws s3 cp s3://my-bucket/my-object /local/dir
aws s3 cp s3://my-bucket/my-object /local/dir --recursive
Code to copy an object from S3.
import boto3
# Initialize a session using Amazon S3
s3 = boto3.client('s3')
# Define the bucket name and object key
bucket_name = 'my-bucket'
object_key = 'my-object'
local_file_path = '/local/dir/my-object' # Adjust the path and filename as needed
# Download the object from S3
try:
s3.download_file(bucket_name, object_key, local_file_path)
print(f'Successfully downloaded {object_key} from bucket {bucket_name} to {local_file_path}')
except Exception as e:
print(f'Error downloading file: {e}')
Code to copy multiple objects from S3.
import os
import boto3
# Initialize a session using Amazon S3
s3 = boto3.client('s3')
# Define the bucket name and prefix (folder) if any
bucket_name = 'my-bucket'
prefix = 'my-folder/' # If you want to download everything, set prefix to ''
# Local directory to save the downloaded files
local_dir = '/local/dir'
# Ensure the local directory exists
if not os.path.exists(local_dir):
os.makedirs(local_dir)
# List objects in the specified bucket and prefix
paginator = s3.get_paginator('list_objects_v2')
pages = paginator.paginate(Bucket=bucket_name, Prefix=prefix)
for page in pages:
if 'Contents' in page:
for obj in page['Contents']:
key = obj['Key']
# Construct local file path
local_file_path = os.path.join(local_dir, key[len(prefix):])
local_file_dir = os.path.dirname(local_file_path)
# Ensure the local directory exists
if not os.path.exists(local_file_dir):
os.makedirs(local_file_dir)
# Download the file
try:
s3.download_file(bucket_name, key, local_file_path)
print(f'Successfully downloaded {key} to {local_file_path}')
except Exception as e:
print(f'Error downloading {key}: {e}')
Resources
Last updated