instanceproperties.template

#################################################################################
#                           SLEEPER INSTANCE PROPERTIES                         #
#################################################################################


## The following instance properties are commonly used throughout Sleeper.

# A string to uniquely identify this deployment. This should be no longer than 20 chars. It should be
# globally unique as it will be used to name AWS resources such as S3 buckets.
# This property may be passed as an argument during deployment. If using using the Sleeper CDK app,
# you can set the context variable "id". If using your own CDK app, you can set this directly.
# (uncomment to set a value)
# sleeper.id=

# The ID of the artefacts deployment to use to deploy the Sleeper instance. By default we assume an
# artefacts deployment with the same ID as the Sleeper instance. This property is used to compute the
# default values of `sleeper.jars.bucket` and `sleeper.ecr.repository.prefix`.
# (uncomment to set a value)
# sleeper.artefacts.deployment=

# The S3 bucket containing the jar files of the Sleeper components. If unset, it will be set by the
# CDK during deployment, based on `sleeper.artefacts.deployment` if it is set, or `sleeper.id` if it
# is not.
# (uncomment to set a value)
# sleeper.jars.bucket=

# If set, this property will be used as a prefix for the names of ECR repositories. If unset, a
# default prefix is computed from `sleeper.artefacts.deployment` if it is set, or `sleeper.id` if it
# is not.
# ECR repository names are generated in the format `<prefix>/<image name>`.
# (uncomment to set a value)
# sleeper.ecr.repository.prefix=

# A comma-separated list of the jars containing application specific iterator code. These jars are
# assumed to be in the bucket given by `sleeper.jars.bucket`. For example, if that bucket contains two
# iterator jars called iterator1.jar and iterator2.jar then the property should be
# 'sleeper.userjars=iterator1.jar,iterator2.jar'.
# (uncomment to set a value)
# sleeper.userjars=

# A comma separated list of resources to exclude from tagging. These should be CDK resource types to
# set in excludeResourceTypes when adding tags, e.g. AWS::EC2::Subnet.
# See the AWS documentation: https://docs.aws.amazon.com/cdk/v2/guide/tagging.html#tagging-props
# (uncomment to set a value)
# sleeper.tags.excluded.resources=

# A name for a tag to identify the stack that deployed a resource. This will be set for all AWS
# resources, to the ID of the CDK stack that they are deployed under. This can be used to organise the
# cost explorer for billing.
# (default value shown below, uncomment to set a value)
# sleeper.stack.tag.name=DeploymentStack

# Whether to keep the Sleeper table bucket, Dynamo tables, query results bucket, etc., when the
# instance is destroyed.
# (default value shown below, uncomment to set a value)
# sleeper.retain.infra.after.destroy=true

# Whether to keep the Sleeper log groups when the instance is destroyed.
# (default value shown below, uncomment to set a value)
# sleeper.retain.logs.after.destroy=true

# This property is used when applying an instance configuration and a table has been removed.
# If this is true (default), removing the table from the configuration will just take the table
# offline.
# If this is false, it will delete all data associated with the table when the table is removed.
# Be aware that if a table is renamed in the configuration, the CDK will see it as a delete of the old
# table name and a create of the new table name. If this is set to false when that happens it will
# remove the table's data.
# This property isn't currently in use but will be in https://github.com/gchq/sleeper/issues/5870.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.retain.after.removal=true

# This property is used when applying an instance configuration and a table has been added.
# By default, or if this property is false, when a table is added to an instance configuration it's
# created in the instance. If it already exists the update will fail.
# If this property is true, the existing table will be reused and imported as part of the instance
# configuration. If it doesn't exist the update will fail.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.reuse.existing=false

# The optional stacks to deploy. Not case sensitive.
# Valid values: [IngestStack, IngestBatcherStack, EmrServerlessBulkImportStack, EmrBulkImportStack,
# PersistentEmrBulkImportStack, EksBulkImportStack, EmrStudioStack, BulkExportStack, QueryStack,
# WebSocketQueryStack, AthenaStack, KeepLambdaWarmStack, CompactionStack, GarbageCollectorStack,
# PartitionSplittingStack, RestApiStack, DashboardStack, TableMetricsStack]
# (default value shown below, uncomment to set a value)
# sleeper.optional.stacks=IngestStack,IngestBatcherStack,EmrServerlessBulkImportStack,EmrStudioStack,QueryStack,CompactionStack,GarbageCollectorStack,PartitionSplittingStack,DashboardStack,TableMetricsStack

# The deployment type for AWS Lambda. Not case sensitive.
# There are two types of Lambda deployments, jar and container.
# If the size of the jar file is too large, it will always be deployed as a container.
# Valid values: [jar, container]
# (default value shown below, uncomment to set a value)
# sleeper.lambda.deploy.type=jar

# The AWS endpoint URL. This should only be set for a non-standard service endpoint. Usually this is
# used to set the URL to LocalStack for a locally deployed instance.
# (uncomment to set a value)
# sleeper.endpoint.url=

# The id of the VPC to deploy to. This property may be passed as an argument during deployment. If
# using the Sleeper CDK app, you can set the context variable "vpc". If using your own CDK app, you
# can set this in SleeperInstanceProps under networking.
# (uncomment to set a value)
# sleeper.vpc=

# Whether to check that the VPC that the instance is deployed to has an S3 endpoint. If there is no S3
# endpoint then the NAT costs can be very significant.
# (default value shown below, uncomment to set a value)
# sleeper.vpc.endpoint.check=true

# A comma separated list of subnets to deploy to. ECS tasks will be run across multiple subnets. EMR
# clusters will be deployed in a subnet chosen when the cluster is created. This property may be
# passed as an argument during deployment. If using the Sleeper CDK app, you can set the context
# variable "subnets". If using your own CDK app, you can set this in SleeperInstanceProps under
# networking.
# (uncomment to set a value)
# sleeper.subnets=

# The Hadoop filesystem used to connect to S3.
# (default value shown below, uncomment to set a value)
# sleeper.filesystem=s3a://

# An email address used by the TopicStack to publish SNS notifications of errors.
# (uncomment to set a value)
# sleeper.errors.email=

# The length of time in days that CloudWatch logs from lambda functions, ECS containers, etc., are
# retained.
# See https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-logs-loggroup.html
# for valid options.
# Use -1 to indicate infinite retention.
# (default value shown below, uncomment to set a value)
# sleeper.log.retention.days=30

# Used to set the value of fs.s3a.connection.maximum on the Hadoop configuration. This controls the
# maximum number of http connections to S3.
# See https://hadoop.apache.org/docs/stable/hadoop-aws/tools/hadoop-aws/performance.html
# (default value shown below, uncomment to set a value)
# sleeper.fs.s3a.max-connections=100

# Used to set the value of fs.s3a.block.size on the Hadoop configuration. Uploads to S3 happen in
# blocks, and this sets the size of blocks. If a larger value is used, then more data is buffered
# before the upload begins.
# See https://hadoop.apache.org/docs/stable/hadoop-aws/tools/hadoop-aws/performance.html
# (default value shown below, uncomment to set a value)
# sleeper.fs.s3a.upload.block.size=32M

# The version of Fargate to use.
# (default value shown below, uncomment to set a value)
# sleeper.fargate.version=1.4.0

# The amount of memory in MB for the lambda that creates ECS tasks to execute compaction and ingest
# jobs.
# (default value shown below, uncomment to set a value)
# sleeper.task.runner.memory.mb=1024

# The timeout in seconds for the lambda that creates ECS tasks to execute compaction jobs and ingest
# jobs.
# This must be >0 and <= 900.
# (default value shown below, uncomment to set a value)
# sleeper.task.runner.timeout.seconds=900

# If true, properties will be reloaded every time a long running job is started or a lambda is run.
# This will mainly be used in test scenarios to ensure properties are up to date.
# (default value shown below, uncomment to set a value)
# sleeper.properties.force.reload=false

# Default value for the reserved concurrency for each lambda in the Sleeper instance that scales
# according to the number of Sleeper tables.
# The state store committer lambda is an exception to this, as it has reserved concurrency by default.
# This is set in the property sleeper.statestore.committer.concurrency.reserved. Other lambdas are
# present that do not scale by the number of Sleeper tables, and are not set from this property.
# By default no concurrency is reserved for the lambdas. Each lambda also has its own property that
# overrides the value found here.
# See reserved concurrency overview at:
# https://docs.aws.amazon.com/lambda/latest/dg/configuration-concurrency.html
# (uncomment to set a value)
# sleeper.default.lambda.concurrency.reserved=

# Default value for the maximum concurrency for each lambda in the Sleeper instance that scales
# according to the number of Sleeper tables.
# Other lambdas are present that do not scale by the number of Sleeper tables, and are not set from
# this property.
# By default the maximum concurrency is set to 10, which is enough for 10 online tables. If there are
# more online tables, this number may need to be increased. Each lambda also has its own property that
# overrides the value found here.
# See maximum concurrency overview at:
# https://aws.amazon.com/blogs/compute/introducing-maximum-concurrency-of-aws-lambda-functions-when-using-amazon-sqs-as-an-event-source/
# (default value shown below, uncomment to set a value)
# sleeper.default.lambda.concurrency.max=10


## The following instance properties relate to handling the state of Sleeper tables.

# Default value for amount of memory in MB for each lambda that holds the state of Sleeper tables in
# memory. These use a state store provider which caches a number of tables at once, set in
# `sleeper.statestore.provider.cache.size`. Not all lambdas are covered by this, e.g. see
# `sleeper.batch.table.lambdas.memory.mb`.
# (default value shown below, uncomment to set a value)
# sleeper.default.lambda.table.state.memory.mb=4096

# The amount of memory in MB for lambdas that create batches of tables to run some operation against,
# eg. create compaction jobs, run garbage collection, perform partition splitting.
# (default value shown below, uncomment to set a value)
# sleeper.batch.table.lambdas.memory.mb=1024

# The timeout in seconds for lambdas that create batches of tables to run some operation against, eg.
# create compaction jobs, run garbage collection, perform partition splitting.
# (default value shown below, uncomment to set a value)
# sleeper.batch.table.lambdas.timeout.seconds=60

# The timeout in minutes for when the table properties provider cache should be cleared, forcing table
# properties to be reloaded from S3.
# (default value shown below, uncomment to set a value)
# sleeper.cache.table.properties.provider.timeout.minutes=60

# The maximum size of state store providers. If a state store is needed and the cache is full, the
# oldest state store in the cache will be removed to make space. This can be disabled by setting a
# negative value, e.g. -1.
# (default value shown below, uncomment to set a value)
# sleeper.statestore.provider.cache.size=10

# The minimum amount of heap space that the state store provider will try to keep available when
# sizing the cache to the available memory. This will usually only be used in cases where the state
# store cache is the main use of memory, currently only in a state store committer on the EC2
# platform. This affects how many state stores can be cached in memory.
# (default value shown below, uncomment to set a value)
# sleeper.statestore.provider.min.heap.target.amount=100M

# This specifies whether point in time recovery is enabled for the DynamoDB state store. This is set
# on the DynamoDB tables.
# (default value shown below, uncomment to set a value)
# sleeper.statestore.dynamo.pointintimerecovery=false

# This specifies whether point in time recovery is enabled for the S3 state store. This is set on the
# revision DynamoDB table.
# (default value shown below, uncomment to set a value)
# sleeper.statestore.s3.dynamo.pointintimerecovery=false

# The number of tables to create transaction log snapshots for in a single invocation. This will be
# the batch size for a lambda as an SQS FIFO event source. This can be a maximum of 10.
# (default value shown below, uncomment to set a value)
# sleeper.statestore.snapshot.creation.batch.size=1

# The frequency in seconds with which the transaction log snapshot creation lambda is run.
# (default value shown below, uncomment to set a value)
# sleeper.statestore.snapshot.creation.lambda.period.seconds=60

# The timeout in seconds after which to terminate the transaction log snapshot creation lambda.
# (default value shown below, uncomment to set a value)
# sleeper.statestore.snapshot.creation.lambda.timeout.seconds=900

# The amount of memory in MB for the transaction log snapshot creation lambda.
# (default value shown below, uncomment to set a value)
# sleeper.statestore.snapshot.creation.memory.mb=4096

# The reserved concurrency for the snapshot creation lambda.
# See reserved concurrency overview at:
# https://docs.aws.amazon.com/lambda/latest/dg/configuration-concurrency.html
# (uncomment to set a value)
# sleeper.statestore.snapshot.creation.concurrency.reserved=

# The maximum given concurrency allowed for the snapshot creation lambda.
# See maximum concurrency overview at:
# https://aws.amazon.com/blogs/compute/introducing-maximum-concurrency-of-aws-lambda-functions-when-using-amazon-sqs-as-an-event-source/
# (default value shown below, uncomment to set a value)
# sleeper.statestore.snapshot.creation.concurrency.max=10

# The number of tables to delete old transaction log snapshots for in a single invocation. This will
# be the batch size for a lambda as an SQS FIFO event source. This can be a maximum of 10.
# (default value shown below, uncomment to set a value)
# sleeper.statestore.snapshot.deletion.batch.size=1

# The frequency in minutes with which the transaction log snapshot deletion lambda is run.
# (default value shown below, uncomment to set a value)
# sleeper.statestore.snapshot.deletion.lambda.period.minutes=60

# The reserved concurrency for the snapshot deletion lambda.
# See reserved concurrency overview at:
# https://docs.aws.amazon.com/lambda/latest/dg/configuration-concurrency.html
# (uncomment to set a value)
# sleeper.statestore.snapshot.deletion.concurrency.reserved=

# The maximum given concurrency allowed for the snapshot deletion lambda.
# See maximum concurrency overview at:
# https://aws.amazon.com/blogs/compute/introducing-maximum-concurrency-of-aws-lambda-functions-when-using-amazon-sqs-as-an-event-source/
# (default value shown below, uncomment to set a value)
# sleeper.statestore.snapshot.deletion.concurrency.max=10

# The number of tables to delete old transaction log transactions for in a single invocation. This
# will be the batch size for a lambda as an SQS FIFO event source. This can be a maximum of 10.
# (default value shown below, uncomment to set a value)
# sleeper.statestore.transaction.deletion.batch.size=1

# The frequency in minutes with which the transaction log transaction deletion lambda is run.
# (default value shown below, uncomment to set a value)
# sleeper.statestore.transaction.deletion.lambda.period.minutes=60

# The reserved concurrency for the transaction deletion lambda.
# See reserved concurrency overview at:
# https://docs.aws.amazon.com/lambda/latest/dg/configuration-concurrency.html
# (uncomment to set a value)
# sleeper.statestore.transaction.deletion.concurrency.reserved=

# The maximum given concurrency allowed for the transaction deletion lambda.
# See maximum concurrency overview at:
# https://aws.amazon.com/blogs/compute/introducing-maximum-concurrency-of-aws-lambda-functions-when-using-amazon-sqs-as-an-event-source/
# (default value shown below, uncomment to set a value)
# sleeper.statestore.transaction.deletion.concurrency.max=10

# The maximum timeout for the transaction deletion lambda in seconds.
# (default value shown below, uncomment to set a value)
# sleeper.statestore.transaction.deletion.lambda.timeout.seconds=900

# The reserved concurrency for the lambda that follows the state store transaction log to trigger
# updates.
# See reserved concurrency overview at:
# https://docs.aws.amazon.com/lambda/latest/dg/configuration-concurrency.html
# (uncomment to set a value)
# sleeper.statestore.transaction.follower.concurrency.reserved=

# The maximum given concurrency allowed for the lambda that follows the state store transaction log to
# trigger updates.
# See maximum concurrency overview at:
# https://aws.amazon.com/blogs/compute/introducing-maximum-concurrency-of-aws-lambda-functions-when-using-amazon-sqs-as-an-event-source/
# (default value shown below, uncomment to set a value)
# sleeper.statestore.transaction.follower.concurrency.max=10

# The maximum timeout in seconds for the lambda that follows the state store transaction log to
# trigger updates.
# (default value shown below, uncomment to set a value)
# sleeper.statestore.transaction.follower.lambda.timeout.seconds=900

# The amount of memory in MB for the lambda that follows the state store transaction log to trigger
# updates.
# (default value shown below, uncomment to set a value)
# sleeper.statestore.transaction.follower.memory.mb=4096

# This specifies whether point in time recovery is enabled for the Sleeper table index. This is set on
# the DynamoDB tables.
# (default value shown below, uncomment to set a value)
# sleeper.tables.index.dynamo.pointintimerecovery=false

# This specifies whether queries and scans against the table index DynamoDB tables are strongly
# consistent.
# (default value shown below, uncomment to set a value)
# sleeper.tables.index.dynamo.consistent.reads=true

# The platform that the state store committer will be deployed to for execution.
# Valid values are: [lambda, ec2]
# NB: The EC2 platform is currently considered experimental.
# (default value shown below, uncomment to set a value)
# sleeper.statestore.committer.platform=LAMBDA

# The amount of memory in MB for the lambda that commits state store updates.
# (default value shown below, uncomment to set a value)
# sleeper.statestore.committer.lambda.memory.mb=4096

# The timeout for the lambda that commits state store updates in seconds.
# (default value shown below, uncomment to set a value)
# sleeper.statestore.committer.lambda.timeout.seconds=900

# The number of state store updates to be sent to the state store committer lambda in one invocation.
# This will be the batch size for a lambda as an SQS FIFO event source. This can be a maximum of 10.
# (default value shown below, uncomment to set a value)
# sleeper.statestore.committer.batch.size=10

# The reserved concurrency for the state store committer lambda.
# Presently this value defaults to 10 to align with expectations around table efficiency.
# This is to ensure that state store operations can still be applied to at least 10 tables, even when
# concurrency is used up in the account.
# See reserved concurrency overview at:
# https://docs.aws.amazon.com/lambda/latest/dg/configuration-concurrency.html
# (default value shown below, uncomment to set a value)
# sleeper.statestore.committer.concurrency.reserved=10

# The maximum given concurrency allowed for the state store committer lambda.
# See maximum concurrency overview at:
# https://aws.amazon.com/blogs/compute/introducing-maximum-concurrency-of-aws-lambda-functions-when-using-amazon-sqs-as-an-event-source/
# (default value shown below, uncomment to set a value)
# sleeper.statestore.committer.concurrency.max=10

# The EC2 instance type that the multi-threaded state store committer should be deployed onto.
# (default value shown below, uncomment to set a value)
# sleeper.statestore.committer.ec2.type=m8g.xlarge


## The following instance properties relate to standard ingest.

# The maximum number of concurrent ECS tasks to run.
# (default value shown below, uncomment to set a value)
# sleeper.ingest.max.concurrent.tasks=200

# The frequency in minutes with which an EventBridge rule runs to trigger a lambda that, if necessary,
# runs more ECS tasks to perform ingest jobs.
# (default value shown below, uncomment to set a value)
# sleeper.ingest.task.creation.period.minutes=1

# The frequency, in seconds, with which change message visibility requests are sent to extend the
# visibility of messages on the ingest queue so that they are not processed by other processes.
# This should be less than the value of sleeper.ingest.queue.visibility.timeout.seconds.
# (default value shown below, uncomment to set a value)
# sleeper.ingest.keepalive.period.seconds=300

# The visibility timeout in seconds for the standard ingest job queue. This should be greater than
# sleeper.ingest.keepalive.period.seconds.
# (default value shown below, uncomment to set a value)
# sleeper.ingest.queue.visibility.timeout.seconds=900

# This sets the value of fs.s3a.experimental.input.fadvise on the Hadoop configuration used to read
# and write files to and from S3 in ingest jobs. Changing this value allows you to fine-tune how files
# are read. Possible values are "normal", "sequential" and "random". More information is available
# here:
# https://hadoop.apache.org/docs/current/hadoop-aws/tools/hadoop-aws/performance.html#fadvise.
# (default value shown below, uncomment to set a value)
# sleeper.ingest.fs.s3a.experimental.input.fadvise=sequential

# The amount of CPU used by Fargate tasks that perform ingest jobs.
# Note that only certain combinations of CPU and memory are valid.
# See https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html for valid
# options.
# (default value shown below, uncomment to set a value)
# sleeper.ingest.task.cpu=2048

# The amount of memory in MB used by Fargate tasks that perform ingest jobs.
# Note that only certain combinations of CPU and memory are valid.
# See https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html for valid
# options.
# (default value shown below, uncomment to set a value)
# sleeper.ingest.task.memory.mb=4096

# The frequency in seconds with which ingest tasks refresh their view of the partitions.
# (NB Refreshes only happen once a batch of data has been written so this is a lower bound on the
# refresh frequency.)
# (default value shown below, uncomment to set a value)
# sleeper.ingest.partition.refresh.period=120

# A comma-separated list of bucket names that contain files to be ingested in ingest or bulk import
# jobs. Use this to specify your own buckets that already exist in the same AWS account. Sleeper will
# not create these. The Sleeper CDK deployment will assign permissions to the ingest and bulk import
# systems so that they can consume data from these buckets.
# (uncomment to set a value)
# sleeper.ingest.source.bucket=

# Flag to enable/disable storage of tracking information for ingest jobs and tasks.
# (default value shown below, uncomment to set a value)
# sleeper.ingest.tracker.enabled=true

# The time to live in seconds for ingest job updates in the job tracker. Default is 1 week.
# The expiry time is fixed when an update is saved to the store, so changing this will only affect new
# data.
# (default value shown below, uncomment to set a value)
# sleeper.ingest.job.status.ttl=604800

# The time to live in seconds for ingest task updates in the job tracker. Default is 1 week.
# The expiry time is fixed when an update is saved to the store, so changing this will only affect new
# data.
# (default value shown below, uncomment to set a value)
# sleeper.ingest.task.status.ttl=604800

# The time in seconds to wait for ingest jobs to appear on the queue before an ingest task terminates.
# Must be >= 0 and <= 20.
# See also
# https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-short-and-long-polling.html
# (default value shown below, uncomment to set a value)
# sleeper.ingest.job.queue.wait.time=20

# The maximum number of rows written to local file in an ingest job. Rows are written in sorted order
# to local disk before being uploaded to S3. Increasing this value increases the amount of time before
# data is visible in the system, but increases the number of rows written to S3 in a batch, therefore
# reducing costs.
# (arraylist-based ingest only)
# (default value shown below, uncomment to set a value)
# sleeper.ingest.max.local.rows=100000000

# The maximum number of rows to read into memory in an ingest job. (Up to
# sleeper.ingest.memory.max.batch.size rows are read into memory before being sorted and written to
# disk. This process is repeated until sleeper.ingest.max.local.rows rows have been written to local
# files. Then the sorted files and merged and the data is written to sorted files in S3.)
# (arraylist-based ingest only)
# (default value shown below, uncomment to set a value)
# sleeper.ingest.memory.max.batch.size=1000000

# The number of bytes to allocate to the Arrow working buffer. This buffer is used for sorting and
# other sundry activities. Note that this is off-heap memory, which is in addition to the memory
# assigned to the JVM.
# (arrow-based ingest only) [256MB]
# (default value shown below, uncomment to set a value)
# sleeper.ingest.arrow.working.buffer.bytes=268435456

# The number of bytes to allocate to the Arrow batch buffer, which is used to hold the rows before
# they are written to local disk. A larger value means that the local disk holds fewer, larger files,
# which are more efficient to merge together during an upload to S3. Larger values may require a
# larger working buffer. Note that this is off-heap memory, which is in addition to the memory
# assigned to the JVM.
# (arrow-based ingest only) [1GB]
# (default value shown below, uncomment to set a value)
# sleeper.ingest.arrow.batch.buffer.bytes=1073741824

# The maximum number of bytes to store on the local disk before uploading to the main Sleeper store. A
# larger value reduces the number of S3 PUTs that are required to upload thle data to S3 and results
# in fewer files per partition.
# (arrow-based ingest only) [2GB]
# (default value shown below, uncomment to set a value)
# sleeper.ingest.arrow.max.local.store.bytes=2147483648

# The number of rows to write at once into an Arrow file in the local store. A single Arrow file
# contains many of these micro-batches and so this parameter does not significantly affect the final
# size of the Arrow file. Larger values may require a larger working buffer.
# (arrow-based ingest only) [1K]
# (default value shown below, uncomment to set a value)
# sleeper.ingest.arrow.max.single.write.to.file.rows=1024

# The implementation of the async S3 client to use for upload during ingest.
# Valid values are 'java' or 'crt'. This determines the implementation of S3AsyncClient that gets
# used.
# With 'java' it makes a single PutObject request for each file.
# With 'crt' it uses the AWS Common Runtime (CRT) to make multipart uploads.
# Note that the CRT option is recommended. Using the Java option may cause failures if any file is
# >5GB in size, and will lead to the following warning:
# "The provided S3AsyncClient is not an instance of S3CrtAsyncClient, and thus multipart
# upload/download feature is not enabled and resumable file upload is not supported. To benefit from
# maximum throughput, consider using S3AsyncClient.crtBuilder().build() instead."
# (default value shown below, uncomment to set a value)
# sleeper.ingest.async.client.type=crt

# The part size in bytes to use for multipart uploads.
# (CRT client type only) [128MB]
# (default value shown below, uncomment to set a value)
# sleeper.ingest.async.crt.part.size.bytes=134217728

# The target throughput for multipart uploads, in GB/s. Determines how many parts should be uploaded
# simultaneously.
# (CRT client type only)
# (default value shown below, uncomment to set a value)
# sleeper.ingest.async.crt.target.throughput.gbps=10

# The amount of memory in MB for the lambda that receives submitted requests to ingest files.
# (default value shown below, uncomment to set a value)
# sleeper.ingest.batcher.submitter.memory.mb=1024

# The timeout in seconds for the lambda that receives submitted requests to ingest files. Also used to
# define the visibility timeout for the batcher submit queue.
# (default value shown below, uncomment to set a value)
# sleeper.ingest.batcher.submitter.timeout.seconds=20

# The amount of memory in MB for the lambda that creates ingest jobs from submitted file ingest
# requests.
# (default value shown below, uncomment to set a value)
# sleeper.ingest.batcher.job.creation.memory.mb=1024

# The timeout in seconds for the lambda that creates ingest jobs from submitted file ingest requests.
# (default value shown below, uncomment to set a value)
# sleeper.ingest.batcher.job.creation.timeout.seconds=900

# The rate at which the ingest batcher job creation lambda runs (in minutes, must be >=1).
# (default value shown below, uncomment to set a value)
# sleeper.ingest.batcher.job.creation.period.minutes=1


## The following instance properties relate to bulk export.

# The amount of memory in MB for lambda functions that start bulk export jobs.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.export.memory.mb=4096

# The default timeout in seconds for the bulk export lambda.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.export.timeout.seconds=900

# The visibility timeout in seconds for the bulk export queue.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.export.queue.visibility.timeout.seconds=900

# The CPU architecture to run bulk export tasks on. Valid values are X86_64 and ARM64.
# See Task CPU architecture at
# https://docs.aws.amazon.com/AmazonECS/latest/developerguide/AWS_Fargate.html
# (default value shown below, uncomment to set a value)
# sleeper.bulk.export.task.cpu.architecture=X86_64

# The CPU for a bulk. export task using an ARM64 architecture.
# See https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html for valid
# options.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.export.task.arm.cpu=1024

# The amount of memory in MB for a bulk export task using an ARM64 architecture.
# See https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html for valid
# options.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.export.task.arm.memory.mb=4096

# The CPU for a bulk export task using an x86_64 architecture.
# See https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html for valid
# options.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.export.task.x86.cpu=1024

# The amount of memory in MB for a bulk export task using an x86_64 architecture.
# See https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html for valid
# options.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.export.task.x86.memory.mb=4096

# The rate at which a check to see if bulk export ECS tasks need to be created is made (in minutes,
# must be >= 1).
# (default value shown below, uncomment to set a value)
# sleeper.bulk.export.task.creation.period.minutes=1

# The maximum number of concurrent bulk export tasks to run.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.export.max.concurrent.tasks=300

# The number of days the results of bulk export remain in the bulk export results bucket before being
# deleted.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.export.results.bucket.expiry.days=7

# The delay in seconds until a failed bulk export job becomes visible on the bulk export queue and can
# be processed again.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.export.job.failed.visibility.timeout.seconds=60

# The time in seconds for a bulk export task to wait for a bulk export job to appear on the SQS queue
# (must be <= 20).
# When a bulk export task waits for bulk export jobs to appear on the SQS queue, if the task receives
# no messages in the time defined by this property, it will try to wait for a message again.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.export.task.wait.time.seconds=20

# The frequency, in seconds, with which change message visibility requests are sent to extend the
# visibility of messages on the bulk export job queue so that they are not processed by other
# processes.
# This should be less than the value of sleeper.bulk.export.queue.visibility.timeout.seconds.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.export.keepalive.period.seconds=300


## The following instance properties relate to bulk import, i.e. ingesting data using Spark jobs
## running on EMR or EKS.
## 
## Note that on EMR, the total resource allocation must align with the instance types used for the
## cluster. For the maximum memory usage, combine the memory and memory overhead properties, and
## compare against the maximum memory allocation for YARN in the Hadoop task configuration:
## 
## https://docs.aws.amazon.com/emr/latest/ReleaseGuide/emr-hadoop-task-config.html
## 
## As an example, if we use m7i.xlarge for executor instances, that has a maximum allocation of 54272
## MiB, or 53 GiB. If we want 3 executors per instance, we can have 53 GiB / 3 = 18,090.666 MiB per
## executor. We can set the executor memory to 16 GiB, and the executor memory overhead to the
## remainder of that amount, which is 18,090 MiB - 16 GiB = 1,706 MiB, or 1.666 GiB. This is just above
## the default Spark memory overhead factor of 0.1, i.e. 16 GiB x 0.1 = 1.6 GiB.
## 
## Also see EMR best practices:
## 
## https://aws.github.io/aws-emr-best-practices/docs/bestpractices/Applications/Spark/best_practices/#bp-516----tune-driverexecutor-memory-cores-and-sparksqlshufflepartitions-to-fully-utilize-cluster-resources

# The class to use to perform the bulk import. The default value below uses Spark Dataframes. There is
# an alternative option that uses RDDs (sleeper.bulkimport.runner.rdd.BulkImportJobRDDDriver).
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.class.name=sleeper.bulkimport.runner.dataframelocalsort.BulkImportDataframeLocalSortDriver

# The compression codec for map status results. Used to set spark.shuffle.mapStatus.compression.codec.
# Stops "Decompression error: Version not supported" errors - only a value of "lz4" has been tested.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.spark.shuffle.mapStatus.compression.codec=lz4

# If true then speculative execution of tasks will be performed. Used to set spark.speculation.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.spark.speculation=false

# Fraction of tasks which must be complete before speculation is enabled for a particular stage. Used
# to set spark.speculation.quantile.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.spark.speculation.quantile=0.75

# The amount of memory in MB for lambda functions that start bulk import jobs.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.starter.memory.mb=4096

# The number of days a bulk import job is held in the bulk import bucket. When a job is submitted to
# Spark it is written to S3 for Spark to read. This is retained in case the job needs to be retried,
# and for diagnostic purposes.
# Defaults to the value of sleeper.log.retention.days.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.job.file.retention.days=30

# The amount of memory allocated to a Spark executor. Used to set spark.executor.memory.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.spark.executor.memory=16g

# The amount of memory allocated to the Spark driver. Used to set spark.driver.memory.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.spark.driver.memory=16g

# The number of executors. Used to set spark.executor.instances.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.spark.executor.instances=29

# The memory overhead for an executor. Used to set spark.executor.memoryOverhead.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.spark.executor.memory.overhead=1706m

# The memory overhead for the driver. Used to set spark.driver.memoryOverhead.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.spark.driver.memory.overhead=1706m

# The default parallelism for Spark job. Used to set spark.default.parallelism.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.spark.default.parallelism=290

# The number of partitions used in a Spark SQL/dataframe shuffle operation. Used to set
# spark.sql.shuffle.partitions.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.spark.sql.shuffle.partitions=290

# (Non-persistent or persistent EMR mode only) An EC2 keypair to use for the EC2 instances. Specifying
# this will allow you to SSH to the nodes in the cluster while it's running. NOTE: Updating this
# property will cause any existing EMR cluster to be destroyed and re-created.
# (uncomment to set a value)
# sleeper.bulk.import.emr.keypair.name=

# (Non-persistent or persistent EMR mode only) Specifying this security group causes the group to be
# added to the EMR master's list of security groups.
# (uncomment to set a value)
# sleeper.bulk.import.emr.master.additional.security.group=

# (Non-persistent or persistent EMR mode only) The number of cores used by an executor. Used to set
# spark.executor.cores.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.spark.executor.cores=4

# (Non-persistent or persistent EMR mode only) The number of cores used by the driver. Used to set
# spark.driver.cores.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.spark.driver.cores=4

# (Non-persistent or persistent EMR mode only) The default timeout for network interactions in Spark.
# Used to set spark.network.timeout.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.spark.network.timeout=800s

# (Non-persistent or persistent EMR mode only) The interval between heartbeats from executors to the
# driver. Used to set spark.executor.heartbeatInterval.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.spark.executor.heartbeat.interval=60s

# (Non-persistent or persistent EMR mode only) Whether Spark should use dynamic allocation to scale
# resources up and down. Used to set spark.dynamicAllocation.enabled.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.spark.dynamic.allocation.enabled=false

# (Non-persistent or persistent EMR mode only) The fraction of heap space used for execution and
# storage. Used to set spark.memory.fraction.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.spark.memory.fraction=0.80

# (Non-persistent or persistent EMR mode only) The amount of storage memory immune to eviction,
# expressed as a fraction of the heap space used for execution and storage. Used to set
# spark.memory.storageFraction.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.spark.memory.storage.fraction=0.30

# (Non-persistent or persistent EMR mode only) JVM options passed to the executors. Used to set
# spark.executor.extraJavaOptions.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.spark.executor.extra.java.options=-XX:+UseG1GC -XX:+UnlockDiagnosticVMOptions -XX:+G1SummarizeConcMark -XX:InitiatingHeapOccupancyPercent=35 -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:OnOutOfMemoryError='kill -9 %p'

# (Non-persistent or persistent EMR mode only) JVM options passed to the driver. Used to set
# spark.driver.extraJavaOptions.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.spark.driver.extra.java.options=-XX:+UseG1GC -XX:+UnlockDiagnosticVMOptions -XX:+G1SummarizeConcMark -XX:InitiatingHeapOccupancyPercent=35 -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:OnOutOfMemoryError='kill -9 %p'

# (Non-persistent or persistent EMR mode only) The maximum number of executor failures before YARN can
# fail the application. Used to set spark.yarn.scheduler.reporterThread.maxFailures.
# See
# https://aws.amazon.com/blogs/big-data/best-practices-for-successfully-managing-memory-for-apache-spark-applications-on-amazon-emr/.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.spark.yarn.scheduler.reporter.thread.max.failures=5

# (Non-persistent or persistent EMR mode only) The storage to use for temporary caching. Used to set
# spark.storage.level.
# See
# https://aws.amazon.com/blogs/big-data/best-practices-for-successfully-managing-memory-for-apache-spark-applications-on-amazon-emr/.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.spark.storage.level=MEMORY_AND_DISK_SER

# (Non-persistent or persistent EMR mode only) Whether to compress serialized RDD partitions. Used to
# set spark.rdd.compress.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.spark.rdd.compress=true

# (Non-persistent or persistent EMR mode only) Whether to compress map output files. Used to set
# spark.shuffle.compress.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.spark.shuffle.compress=true

# (Non-persistent or persistent EMR mode only) Whether to compress data spilled during shuffles. Used
# to set spark.shuffle.spill.compress.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.spark.shuffle.spill.compress=true

# (Non-persistent or persistent EMR mode only) The size of the EBS volume in gibibytes (GiB).
# This can be a number from 10 to 1024.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.ebs.volume.size.gb=256

# (Non-persistent or persistent EMR mode only) The type of the EBS volume.
# Valid values are 'gp2', 'gp3', 'io1', 'io2'.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.ebs.volume.type=gp2

# (Non-persistent or persistent EMR mode only) The number of EBS volumes per instance.
# This can be a number from 1 to 25.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.ebs.volumes.per.instance=4

# ARN of the KMS Key used to encrypt data at rest on the local file system in AWS EMR.
# See
# https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-encryption-enable.html#emr-encryption-create-keys.
# (uncomment to set a value)
# sleeper.bulk.import.emr.ebs.encryption.key.arn=

# The architecture for EMR Serverless to use. X86_64 or ARM64 (Coming soon)
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.serverless.architecture=X86_64

# The version of EMR Serverless to use.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.serverless.release=emr-7.12.0

# Set to true to allow an EMR Serverless Application to start automatically when a job is submitted.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.serverless.autostart.enabled=true

# Set to true to allow an EMR Serverless Application to stop automatically when there are no jobs to
# process.
# Turning this off with pre-initialised capacity turned off is not recommended.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.serverless.autostop.enabled=true

# The number of minutes of inactivity before EMR Serverless stops the application.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.serverless.autostop.timeout=15

# The number of cores used by a Serverless executor. Used to set spark.executor.cores.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.serverless.spark.executor.cores=4

# The amount of memory allocated to a Serverless executor. Used to set spark.executor.memory.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.serverless.spark.executor.memory=16G

# The amount of storage allocated to a Serverless executor.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.serverless.spark.emr-serverless.executor.disk=200G

# The number of executors to be used with Serverless. Used to set spark.executor.instances.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.serverless.spark.executor.instances=36

# The number of cores used by the Serverless Spark driver. Used to set spark.driver.cores.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.serverless.spark.driver.cores=4

# The amount of memory allocated to the Serverless Spark driver. Used to set spark.driver.memory.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.serverless.spark.driver.memory=16G

# The path to JAVA_HOME to be used by the custom image for bulk import.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.serverless.spark.executorEnv.JAVA_HOME=/usr/lib/jvm/jre-11

# Whether Spark should use dynamic allocation to scale resources up and down. Used to set
# spark.dynamicAllocation.enabled. See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.serverless.spark.dynamic.allocation.enabled=false

# Whether to compress serialized RDD partitions. Used to set spark.rdd.compress.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.serverless.spark.rdd.compress=true

# Whether to compress map output files. Used to set spark.shuffle.compress.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.serverless.spark.shuffle.compress=true

# Whether to compress data spilled during shuffles. Used to set spark.shuffle.spill.compress.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.serverless.spark.shuffle.spill.compress=true

# The default parallelism for Spark job. Used to set spark.default.parallelism.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.serverless.spark.default.parallelism=288

# The number of partitions used in a Spark SQL/dataframe shuffle operation. Used to set
# spark.sql.shuffle.partitions.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.serverless.spark.sql.shuffle.partitions=288

# The default timeout for network interactions in Spark. Used to set spark.network.timeout.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.serverless.spark.network.timeout=800s

# (The interval between heartbeats from executors to the driver. Used to set
# spark.executor.heartbeatInterval.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.serverless.spark.executor.heartbeat.interval=60s

# The fraction of heap space used for execution and storage. Used to set spark.memory.fraction.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.serverless.spark.memory.fraction=0.80

# The amount of storage memory immune to eviction, expressed as a fraction of the heap space used for
# execution and storage. Used to set spark.memory.storageFraction.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.serverless.spark.memory.storage.fraction=0.30

# If true then speculative execution of tasks will be performed. Used to set spark.speculation.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.serverless.spark.speculation=false

# Fraction of tasks which must be complete before speculation is enabled for a particular stage. Used
# to set spark.speculation.quantile.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.serverless.spark.speculation.quantile=0.75

# The compression codec for map status results. Used to set spark.shuffle.mapStatus.compression.codec.
# Stops "Decompression error: Version not supported" errors - only a value of "lz4" has been tested.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.serverless.spark.shuffle.mapStatus.compression.codec=lz4

# Set to enable the pre-initialise capacity option for EMR Serverless application.
# See: https://docs.aws.amazon.com/emr/latest/EMR-Serverless-UserGuide/pre-init-capacity.html
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.serverless.initial.capacity.enabled=false

# The number of executors to pre-initialise.
# See: https://docs.aws.amazon.com/emr/latest/EMR-Serverless-UserGuide/pre-init-capacity.html
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.serverless.initial.capacity.executor.count=72

# The amount of CPUs per executor for the pre-initialise capacity.
# See: https://docs.aws.amazon.com/emr/latest/EMR-Serverless-UserGuide/pre-init-capacity.html
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.serverless.initial.capacity.executor.cores=4vCPU

# The amount of memory per executor for the pre-initialise capacity.
# See: https://docs.aws.amazon.com/emr/latest/EMR-Serverless-UserGuide/pre-init-capacity.html
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.serverless.initial.capacity.executor.memory=18GB

# The amount of storage per executor for the pre-initialise capacity.
# See: https://docs.aws.amazon.com/emr/latest/EMR-Serverless-UserGuide/pre-init-capacity.html
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.serverless.initial.capacity.executor.disk=200GB

# The number of drivers to pre-initialise.
# See: https://docs.aws.amazon.com/emr/latest/EMR-Serverless-UserGuide/pre-init-capacity.html
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.serverless.initial.capacity.driver.count=5

# The amount of CPUs per driver for the pre-initialise capacity.
# See: https://docs.aws.amazon.com/emr/latest/EMR-Serverless-UserGuide/pre-init-capacity.html
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.serverless.initial.capacity.driver.cores=4vCPU

# The amount of memory per driver for the pre-initialise capacity.
# See: https://docs.aws.amazon.com/emr/latest/EMR-Serverless-UserGuide/pre-init-capacity.html
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.serverless.initial.capacity.driver.memory=18GB

# The amount of storage per driver for the pre-initialise capacity.
# See: https://docs.aws.amazon.com/emr/latest/EMR-Serverless-UserGuide/pre-init-capacity.html
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.emr.serverless.initial.capacity.driver.disk=20GB

# (Non-persistent EMR mode only) The default EMR release label to be used when creating an EMR cluster
# for bulk importing data using Spark running on EMR.
# This property is a default which can be overridden by a table property or by a property in the bulk
# import job specification.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.bulk.import.emr.release.label=emr-7.12.0

# (Non-persistent EMR mode only) Which architecture to be used for EC2 instance types in the EMR
# cluster. Must be either "x86_64" "arm64" or "x86_64,arm64". For more information, see the Bulk
# import using EMR - Instance types section in docs/usage/bulk-import.md
# (default value shown below, uncomment to set a value)
# sleeper.default.table.bulk.import.emr.instance.architecture=arm64

# (Non-persistent EMR mode only) The default EC2 x86_64 instance types and weights to be used for the
# master node of the EMR cluster.
# For more information, see the Bulk import using EMR - Instance types section in
# docs/usage/bulk-import.md
# (default value shown below, uncomment to set a value)
# sleeper.default.table.bulk.import.emr.master.x86.instance.types=m7i.xlarge

# (Non-persistent EMR mode only) The default EC2 x86_64 instance types and weights to be used for the
# executor nodes of the EMR cluster.
# For more information, see the Bulk import using EMR - Instance types section in
# docs/usage/bulk-import.md
# (default value shown below, uncomment to set a value)
# sleeper.default.table.bulk.import.emr.executor.x86.instance.types=m7i.4xlarge

# (Non-persistent EMR mode only) The default EC2 ARM64 instance types and weights to be used for the
# master node of the EMR cluster.
# For more information, see the Bulk import using EMR - Instance types section in
# docs/usage/bulk-import.md
# (default value shown below, uncomment to set a value)
# sleeper.default.table.bulk.import.emr.master.arm.instance.types=m7g.xlarge

# (Non-persistent EMR mode only) The default EC2 ARM64 instance types and weights to be used for the
# executor nodes of the EMR cluster.
# For more information, see the Bulk import using EMR - Instance types section in
# docs/usage/bulk-import.md
# (default value shown below, uncomment to set a value)
# sleeper.default.table.bulk.import.emr.executor.arm.instance.types=m7g.4xlarge

# (Non-persistent EMR mode only) The default purchasing option to be used for the executor nodes of
# the EMR cluster.
# Valid values are ON_DEMAND or SPOT.
# This property is a default which can be overridden by a table property or by a property in the bulk
# import job specification.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.bulk.import.emr.executor.market.type=SPOT

# (Non-persistent EMR mode only) The default initial number of capacity units to provision as EC2
# instances for executors in the EMR cluster.
# This is measured in instance fleet capacity units. These are declared alongside the requested
# instance types, as each type will count for a certain number of units. By default the units are the
# number of instances.
# This property is a default which can be overridden by a table property or by a property in the bulk
# import job specification.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.bulk.import.emr.executor.initial.capacity=2

# (Non-persistent EMR mode only) The default maximum number of capacity units to provision as EC2
# instances for executors in the EMR cluster.
# This is measured in instance fleet capacity units. These are declared alongside the requested
# instance types, as each type will count for a certain number of units. By default the units are the
# number of instances.
# This property is a default which can be overridden by a table property or by a property in the bulk
# import job specification.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.bulk.import.emr.executor.max.capacity=10

# (Persistent EMR mode only) The EMR release used to create the persistent EMR cluster.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.persistent.emr.release.label=emr-7.12.0

# (Persistent EMR mode only) Which architecture to be used for EC2 instance types in the EMR cluster.
# Must be either "x86_64" "arm64" or "x86_64,arm64". For more information, see the Bulk import using
# EMR - Instance types section in docs/usage/bulk-import.md
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.persistent.emr.instance.architecture=arm64

# (Persistent EMR mode only) The EC2 x86_64 instance types and weights used for the master node of the
# persistent EMR cluster.
# For more information, see the Bulk import using EMR - Instance types section in
# docs/usage/bulk-import.md
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.persistent.emr.master.x86.instance.types=m7i.xlarge

# (Persistent EMR mode only) The EC2 x86_64 instance types and weights used for the executor nodes of
# the persistent EMR cluster.
# For more information, see the Bulk import using EMR - Instance types section in
# docs/usage/bulk-import.md
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.persistent.emr.executor.x86.instance.types=m7i.4xlarge

# (Persistent EMR mode only) The EC2 ARM64 instance types and weights used for the master node of the
# persistent EMR cluster.
# For more information, see the Bulk import using EMR - Instance types section in
# docs/usage/bulk-import.md
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.persistent.emr.master.arm.instance.types=m7g.xlarge

# (Persistent EMR mode only) The EC2 ARM64 instance types and weights used for the executor nodes of
# the persistent EMR cluster.
# For more information, see the Bulk import using EMR - Instance types section in
# docs/usage/bulk-import.md
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.persistent.emr.executor.arm.instance.types=m7g.4xlarge

# (Persistent EMR mode only) Whether the persistent EMR cluster should use managed scaling or not.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.persistent.emr.use.managed.scaling=true

# (Persistent EMR mode only) The minimum number of capacity units to provision as EC2 instances for
# executors in the persistent EMR cluster.
# This is measured in instance fleet capacity units. These are declared alongside the requested
# instance types, as each type will count for a certain number of units. By default the units are the
# number of instances.
# If managed scaling is not used then the cluster will be of fixed size, with a number of instances
# equal to this value. This value must be in the range [1, 2000].
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.persistent.emr.min.capacity=1

# (Persistent EMR mode only) The maximum number of capacity units to provision as EC2 instances for
# executors in the persistent EMR cluster.
# This is measured in instance fleet capacity units. These are declared alongside the requested
# instance types, as each type will count for a certain number of units. By default the units are the
# number of instances.
# This value is only used if managed scaling is used. This value must be in the range [1, 2000] and
# greater than
# the minimum capacity.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.persistent.emr.max.capacity=10

# (Persistent EMR mode only) This controls the number of EMR steps that can run concurrently.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.persistent.emr.step.concurrency.level=2

# (Persistent EMR mode only) The number of seconds to wait before requeueing a bulk import job because
# the persistent EMR cluster is full.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.persistent.emr.cluster.full.requeue.delay=60

# (EKS mode only) Names of AWS IAM roles which should have access to administer the EKS cluster.
# (uncomment to set a value)
# sleeper.bulk.import.eks.cluster.admin.roles=

# (EKS mode only) IDs of security groups that should be permitted to reach the EKS Kubernetes API.
# Each ID listed here is added as an ingress rule on TCP/443 to the cluster's security group.
# Required if you want to call the API from another host inside the cluster's VPC, since the cluster's
# default security group only permits traffic from the cluster itself. Leave unset for deployments
# where only the cluster's own components need API access, or where the API accessed from outside the
# VPC.
# (uncomment to set a value)
# sleeper.bulk.import.eks.api.allowed.security.groups=

# (EKS mode only) Set to true if sleeper.bulk.import.eks.repo contains the image built with native
# Hadoop libraries. By default when deploying with the EKS stack enabled, an image will be built based
# on the official Spark Docker image, so this should be false.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.eks.is.native.libs.image=false

# (EKS mode only) The ARN of a Lambda Layer providing the AWS CLI to use with the EKS kubectl
# provider. If not set, the default AWS CLI layer included with the CDK will be used.
# (uncomment to set a value)
# sleeper.bulk.import.eks.awscli.layer.arn=

# (EKS mode only) The type of EKS cluster to deploy for bulk import.
# Valid values are: [fargate, automode]
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.eks.cluster.type=FARGATE

# (EKS mode only, automode cluster type only) Whether to configure the node pool for the cluster. If
# this is enabled, related properties will be applied. Otherwise it will use the default behaviour for
# EKS Auto Mode.
# Node pool configuration is currently experimental.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.eks.automode.nodepool.enabled=false

# (EKS mode only, automode cluster type only) Comma-separated list of AWS EC2 instance types that the
# Karpenter NodePool is allowed to launch for Spark pods.
# Only applied if node pool configuration is enabled. Currently experimental.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.eks.automode.nodepool.instance.types=m8g.xlarge,m8g.2xlarge,m8g.4xlarge,m8g.8xlarge,m8g.12xlarge,m8g.16xlarge,m8gd.xlarge,m8gd.2xlarge,m8gd.4xlarge,m8gd.8xlarge,m8gd.12xlarge,m8gd.16xlarge,m8i.xlarge,m8i.2xlarge,m8i.4xlarge,m8i.8xlarge,m8i.12xlarge,m8i.16xlarge,m8id.xlarge,m8id.2xlarge,m8id.4xlarge,m8id.8xlarge,m8id.12xlarge,m8id.16xlarge

# (EKS mode only, automode cluster type only) The maximum total number of CPU cores the Karpenter
# NodePool is allowed to provision across all nodes. Must be an integer greater than 0.
# The default value assumes 2 concurrent jobs, running on 29 executors and 1 driver, each with 4
# cores, plus one submitter per job and some slack.
# Only applied if node pool configuration is enabled. Currently experimental.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.eks.automode.nodepool.cpu.limit=256

# (EKS mode only) This controls the number of Kubernetes jobs that can run concurrently in the bulk
# import namespace. Enforced by a ResourceQuota on count/jobs.batch.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.eks.job.concurrency.level=2

# (EKS auto mode only) Whether to deploy a FluentBit DaemonSet to collect container logs from EKS auto
# mode nodes and send them to CloudWatch Logs. Disable this if you manage log collection separately or
# need to avoid the aws-for-fluent-bit image dependency.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.eks.automode.fluentbit.logging.enabled=true

# (EKS mode only) The number of Spark executors. Used to set spark.executor.instances.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.eks.spark.executor.instances=29

# (EKS mode only) The amount of memory allocated to the Spark driver. Used to set spark.driver.memory.
# Default values are overridden because Fargate doesn't work with Spark's default values.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.eks.spark.driver.memory=16g

# (EKS mode only) The amount of memory allocated to a Spark executor. Used to set
# spark.executor.memory. Default values are overridden because Fargate doesn't work with Spark's
# default values.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.eks.spark.executor.memory=16g

# (EKS mode only) The memory overhead for the Spark driver. Used to set spark.driver.memoryOverhead.
# Fargate provides extra memory so no need to include extra which also messes up the scheduler.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.eks.spark.driver.memory.overhead=1706m

# (EKS mode only) The memory overhead for a Spark executor. Used to set spark.executor.memoryOverhead.
# Fargate provides extra memory so no need to include extra which also messes up the scheduler.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.eks.spark.executor.memory.overhead=1706m

# (EKS mode only) The S3A input read policy. Used to set
# spark.hadoop.fs.s3a.experimental.input.fadvise.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.eks.spark.hadoop.fs.s3a.experimental.input.fadvise=sequential

# (EKS mode only) The number of cores used by a Spark executor. Used to set spark.executor.cores.
# Should reflect the Fargate task shape rather than the EMR EC2 instance type.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.eks.spark.executor.cores=4

# (EKS mode only) The number of cores used by the Spark driver. Used to set spark.driver.cores. Should
# reflect the Fargate task shape rather than the EMR EC2 instance type.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.eks.spark.driver.cores=4

# (EKS mode only) The default parallelism for the Spark job. Used to set spark.default.parallelism.
# Should scale with the total cores across the EKS cluster, which may differ from EMR.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.eks.spark.default.parallelism=290

# (EKS mode only) The number of partitions used in a Spark SQL/dataframe shuffle operation. Used to
# set spark.sql.shuffle.partitions.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.eks.spark.sql.shuffle.partitions=290

# (EKS mode only) Whether Spark should use dynamic allocation to scale resources up and down. Used to
# set spark.dynamicAllocation.enabled. Kubernetes support for dynamic allocation is more limited than
# YARN's; consider leaving this disabled on EKS.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.eks.spark.dynamic.allocation.enabled=false

# JVM options passed to the executors. Used to set spark.executor.extraJavaOptions.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.eks.spark.executor.extra.java.options=-XX:+UnlockDiagnosticVMOptions -XX:+G1SummarizeConcMark -XX:InitiatingHeapOccupancyPercent=35 -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:OnOutOfMemoryError='kill -9 %p'

# JVM options passed to the driver. Used to set spark.driver.extraJavaOptions.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.eks.spark.driver.extra.java.options=-XX:+UnlockDiagnosticVMOptions -XX:+G1SummarizeConcMark -XX:InitiatingHeapOccupancyPercent=35 -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:OnOutOfMemoryError='kill -9 %p'

# The interval between heartbeats from executors to the driver. Used to set
# spark.executor.heartbeatInterval.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.eks.spark.executor.heartbeat.interval=60s

# The default timeout for network interactions in Spark. Used to set spark.network.timeout.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.eks.spark.network.timeout=800s

# The fraction of heap space used for execution and storage. Used to set spark.memory.fraction.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.eks.spark.memory.fraction=0.80

# The amount of storage memory immune to eviction, expressed as a fraction of the heap space used for
# execution and storage. Used to set spark.memory.storageFraction.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.eks.spark.memory.storage.fraction=0.30

# The storage to use for temporary caching. Used to set spark.storage.level.
# See https://aws.amazon.com/blogs/containers/best-practices-for-running-spark-on-amazon-eks/
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.eks.spark.storage.level=MEMORY_AND_DISK_SER

# Whether to compress serialized RDD partitions. Used to set spark.rdd.compress.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.eks.spark.rdd.compress=true

# Whether to compress map output files. Used to set spark.shuffle.compress.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.eks.spark.shuffle.compress=true

# Whether to compress data spilled during shuffles. Used to set spark.shuffle.spill.compress.
# See https://spark.apache.org/docs/latest/configuration.html.
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.eks.spark.shuffle.spill.compress=true

# (EKS mode only) The amount of ephemeral storage allocated to each Spark executor pod on Fargate. Set
# as the ephemeral-storage request and limit on the executor container via a Spark pod template, which
# Fargate uses to provision ephemeral storage for the pod beyond the default 20 GiB.
# Accepted values 20Gi - 175Gi. See:
# https://spark.apache.org/docs/latest/running-on-kubernetes.html
# https://docs.aws.amazon.com/eks/latest/userguide/fargate-pod-configuration.html#fargate-storage
# (default value shown below, uncomment to set a value)
# sleeper.bulk.import.eks.spark.executor.ephemeral.storage=175Gi


## The following instance properties relate to the splitting of partitions.

# The frequency in minutes with which the lambda runs to find partitions that need splitting and send
# jobs to the splitting lambda.
# (default value shown below, uncomment to set a value)
# sleeper.partition.splitting.period.minutes=30

# When a partition needs splitting, a partition splitting job is created. This reads in the sketch
# files associated to the files in the partition in order to identify the median. This parameter
# controls the maximum number of files that are read in.
# (default value shown below, uncomment to set a value)
# sleeper.partition.splitting.files.maximum=50

# The number of tables to find partitions to split for in a single invocation. This will be the batch
# size for a lambda as an SQS FIFO event source. This can be a maximum of 10.
# (default value shown below, uncomment to set a value)
# sleeper.partition.splitting.finder.batch.size=1

# The amount of memory in MB for the lambda function used to identify partitions that need to be
# split.
# (default value shown below, uncomment to set a value)
# sleeper.partition.splitting.finder.memory.mb=4096

# The timeout in seconds for the lambda function used to identify partitions that need to be split.
# (default value shown below, uncomment to set a value)
# sleeper.partition.splitting.finder.timeout.seconds=900

# The reserved concurrency for the find partitions to split lambda.
# See reserved concurrency overview at:
# https://docs.aws.amazon.com/lambda/latest/dg/configuration-concurrency.html
# (uncomment to set a value)
# sleeper.partition.splitting.finder.concurrency.reserved=

# The maximum given concurrency allowed for the find partitions to split lambda.
# See maximum concurrency overview at:
# https://aws.amazon.com/blogs/compute/introducing-maximum-concurrency-of-aws-lambda-functions-when-using-amazon-sqs-as-an-event-source/
# (default value shown below, uncomment to set a value)
# sleeper.partition.splitting.finder.concurrency.max=10

# The amount of memory in MB for the lambda function used to split partitions.
# (default value shown below, uncomment to set a value)
# sleeper.partition.splitting.memory.mb=4096

# The timeout in seconds for the lambda function used to split partitions.
# (default value shown below, uncomment to set a value)
# sleeper.partition.splitting.timeout.seconds=900

# The number of lambda instances to reserve from your AWS account's quota for splitting partitions.
# Note that this will not provision instances until they are needed. Each time partition splitting
# runs, a separate lambda invocation will be made for each partition that needs to be split. If the
# reserved concurrency is less than the number of partitions that need to be split across all Sleeper
# tables in the instance, these invocations may queue up.
# (default value shown below, uncomment to set a value)
# sleeper.partition.splitting.reserved.concurrency=10

# This is the default value of the partition splitting threshold. Partitions with more than the
# following number of rows in will be split. This value can be overridden on a per-table basis.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.partition.splitting.threshold=1000000000

# When expanding the partition tree explicitly, this is the default for how many rows are required in
# the input data to be able to split a partition. This will be used when pre-splitting partitions.
# For example, during bulk import when there are too few leaf partitions, the partition tree will be
# extended based on the data in the bulk import job. The bulk import job must contain at least this
# much data per new split point.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.partition.splitting.min.rows=1000

# When expanding the partition tree explicitly, this is the default for a minimum percentage of the
# expected number of rows to split a partition assuming an even distribution of rows.
# For example, during bulk import when there are too few leaf partitions, the partition tree will be
# extended based on the data in the bulk import job. For each current leaf partition, we make a sketch
# of the data from the job that's in that partition. We divide the number of rows in the job's input
# data by the current number of leaf partitions, to get the expected rows per partition. If this
# propery is set to 10, then any partition with less than 10% of the expected rows per partition will
# be ignored when extending the partition tree.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.partition.splitting.min.distribution.percent=10


## The following instance properties relate to garbage collection.

# The frequency in minutes with which the garbage collector lambda is run.
# (default value shown below, uncomment to set a value)
# sleeper.gc.period.minutes=15

# The timeout in seconds for the garbage collector lambda.
# (default value shown below, uncomment to set a value)
# sleeper.gc.lambda.timeout.seconds=840

# The amount of memory in MB for the lambda function used to perform garbage collection.
# (default value shown below, uncomment to set a value)
# sleeper.gc.memory.mb=4096

# The reserved concurrency for the garbage collection lambda.
# See reserved concurrency overview at:
# https://docs.aws.amazon.com/lambda/latest/dg/configuration-concurrency.html
# (uncomment to set a value)
# sleeper.gc.concurrency.reserved=

# The maximum given concurrency allowed for the garbage collection lambda.
# See maximum concurrency overview at:
# https://aws.amazon.com/blogs/compute/introducing-maximum-concurrency-of-aws-lambda-functions-when-using-amazon-sqs-as-an-event-source/
# (default value shown below, uncomment to set a value)
# sleeper.gc.concurrency.max=10

# The number of tables to perform garbage collection for in a single invocation. This will be the
# batch size for a lambda as an SQS FIFO event source. This can be a maximum of 10.
# (default value shown below, uncomment to set a value)
# sleeper.gc.table.batch.size=1

# Whether to perform garbage collection for offline tables.
# (default value shown below, uncomment to set a value)
# sleeper.run.gc.offline=false

# The number of deleted files recorded to the state store in a single commit.
# The garbage collector keeps deleting files as long as there are files to delete in the state store,
# and updates the state store whenever it has deleted this many files.
# (default value shown below, uncomment to set a value)
# sleeper.gc.batch.size=10000

# The maximum number of files that can be deleted per invocation of the garbage collector.
# If a batch of files exceeds this limit, the whole batch will be deleted before terminating.
# This limit is applied separately for each Sleeper table.
# This restriction is placed to avoid reaching the lambda timeout for the garbage collector. If this
# timeout is met, it is most likely to happen whilst deleting a batch of files. This would result in
# files being deleted, but the state store not being updated. Any files caught in such a state will be
# found on the next garbage collector run and deleted again, updating the state store as expected.
# (default value shown below, uncomment to set a value)
# sleeper.gc.files.maximum=750000

# A file will not be deleted until this number of minutes have passed after it has been marked as
# ready for garbage collection. The reason for not deleting files immediately after they have been
# marked as ready for garbage collection is that they may still be in use by queries. This property
# can be overridden on a per-table basis.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.gc.delay.minutes=15


## The following instance properties relate to compactions.

# The number of tables to perform compaction job creation for in a single invocation. This will be the
# batch size for a lambda as an SQS FIFO event source. This can be a maximum of 10.
# (default value shown below, uncomment to set a value)
# sleeper.compaction.job.creation.batch.size=1

# The number of finished compaction commits to gather in the batcher before committing to the state
# store. This will be the batch size for a lambda as an SQS event source.
# This can be a maximum of 10,000. In practice the effective maximum is limited by the number of
# messages that fit in a synchronous lambda invocation payload, see the AWS documentation:
# https://docs.aws.amazon.com/lambda/latest/dg/gettingstarted-limits.html
# (default value shown below, uncomment to set a value)
# sleeper.compaction.job.commit.batch.size=1000

# The time in seconds that the batcher will wait for compaction commits to appear if the batch size is
# not filled. This will be set in the SQS event source for the lambda. This can be a maximum of 300,
# i.e. 5 minutes.
# (default value shown below, uncomment to set a value)
# sleeper.compaction.job.commit.batching.window.seconds=30

# The visibility timeout for the queue of compaction jobs.
# This should be greater than the value of sleeper.compaction.keepalive.period.seconds.
# (default value shown below, uncomment to set a value)
# sleeper.compaction.queue.visibility.timeout.seconds=900

# The visibility timeout for the queue of pending compaction job batches.
# (default value shown below, uncomment to set a value)
# sleeper.compaction.pending.queue.visibility.timeout.seconds=900

# The frequency, in seconds, with which change message visibility requests are sent to extend the
# visibility of messages on the compaction job queue so that they are not processed by other
# processes.
# This should be less than the value of sleeper.compaction.queue.visibility.timeout.seconds.
# (default value shown below, uncomment to set a value)
# sleeper.compaction.keepalive.period.seconds=300

# The delay in seconds until a failed compaction job becomes visible on the compaction job queue and
# can be processed again.
# (default value shown below, uncomment to set a value)
# sleeper.compaction.job.failed.visibility.timeout.seconds=60

# The time in seconds for a compaction task to wait for a compaction job to appear on the SQS queue
# (must be <= 20).
# When a compaction task waits for compaction jobs to appear on the SQS queue, if the task receives no
# messages in the time defined by this property, it will try to wait for a message again.
# (default value shown below, uncomment to set a value)
# sleeper.compaction.task.wait.time.seconds=20

# Set to true if compaction tasks should wait for input files to be assigned to a compaction job
# before starting it. The compaction task will poll the state store for whether the input files have
# been assigned to the job, and will only start once this has occurred.
# This prevents invalid compaction jobs from being run, particularly in the case where the compaction
# job creator runs again before the input files are assigned.
# This also causes compaction tasks to wait idle while input files are assigned, and puts extra load
# on the state store when there are many compaction tasks.
# If this is false, any created job will be executed, and will only be validated when committed to the
# state store.
# (default value shown below, uncomment to set a value)
# sleeper.compaction.task.wait.for.input.file.assignment=false

# The time in seconds for a compaction task to wait after receiving no compaction jobs before
# attempting to receive a message again.
# When a compaction task waits for compaction jobs to appear on the SQS queue, if the task receives no
# messages in the time defined by the property "sleeper.compaction.task.wait.time.seconds", it will
# wait for a number of seconds defined by this property, then try to receive a message again.
# (default value shown below, uncomment to set a value)
# sleeper.compaction.task.delay.before.retry.seconds=10

# The total time in seconds that a compaction task can be idle before it is terminated.
# When there are no compaction jobs available on the SQS queue, and SQS returns no jobs, the task will
# check whether this idle time has elapsed since the last time it finished a job. If so, the task will
# terminate.
# (default value shown below, uncomment to set a value)
# sleeper.compaction.task.max.idle.time.seconds=60

# The total time in minutes that a compaction task can be alive for before it is terminated.
# When the task has finished its current job, it will calculate how long it's been running and if it's
# been running longer than the max alive time it will terminate.
# A new task will then begin running to process remaining compaction jobs.
# This allows the user to gracefully upgrade Sleeper to a newer version.
# (default value shown below, uncomment to set a value)
# sleeper.compaction.task.max.alive.time.minutes=1440

# The max amount of jitter to subtract from a new compaction task's max alive time.
# A random number of minutes between 0 and this value will be subtracted from the max alive time
# property above for each new compaction task.
# This lowers the chance that all the compaction tasks stop at the same time.
# (default value shown below, uncomment to set a value)
# sleeper.compaction.task.max.alive.jitter.minutes=120

# The maximum number of times that a compaction task can fail to process consecutive compaction jobs
# before it terminates.
# When the task starts or completes any job successfully, the count of consecutive failures is set to
# zero. Any time it fails to process a job, this count is incremented. If this maximum is reached, the
# task will terminate.
# (default value shown below, uncomment to set a value)
# sleeper.compaction.task.max.consecutive.failures=3

# The rate at which the compaction job creation lambda runs (in minutes, must be >=1).
# (default value shown below, uncomment to set a value)
# sleeper.compaction.job.creation.period.minutes=1

# The amount of memory in MB for the lambda that creates compaction jobs.
# (default value shown below, uncomment to set a value)
# sleeper.compaction.job.creation.memory.mb=4096

# The timeout for the lambda that creates compaction jobs in seconds.
# (default value shown below, uncomment to set a value)
# sleeper.compaction.job.creation.timeout.seconds=900

# The reserved concurrency for the lambda used to create compaction jobs.
# See reserved concurrency overview at:
# https://docs.aws.amazon.com/lambda/latest/dg/configuration-concurrency.html
# (uncomment to set a value)
# sleeper.compaction.job.creation.concurrency.reserved=

# The maximum given concurrency allowed for the lambda used to create compaction jobs.
# See maximum concurrency overview at:
# https://aws.amazon.com/blogs/compute/introducing-maximum-concurrency-of-aws-lambda-functions-when-using-amazon-sqs-as-an-event-source/
# (default value shown below, uncomment to set a value)
# sleeper.compaction.job.creation.concurrency.max=10

# The amount of memory in MB for the lambda that sends batches of compaction jobs.
# (default value shown below, uncomment to set a value)
# sleeper.compaction.job.dispatch.memory.mb=4096

# The timeout for the lambda that sends batches of compaction jobs in seconds.
# (default value shown below, uncomment to set a value)
# sleeper.compaction.job.dispatch.timeout.seconds=900

# The reserved concurrency for the lambda that sends batches of compaction jobs.
# See reserved concurrency overview at:
# https://docs.aws.amazon.com/lambda/latest/dg/configuration-concurrency.html
# (uncomment to set a value)
# sleeper.compaction.job.dispatch.concurrency.reserved=

# The maximum concurrency allowed for the lambda that sends batches of compaction jobs.
# See maximum concurrency overview at:
# https://aws.amazon.com/blogs/compute/introducing-maximum-concurrency-of-aws-lambda-functions-when-using-amazon-sqs-as-an-event-source/
# (default value shown below, uncomment to set a value)
# sleeper.compaction.job.dispatch.concurrency.max=10

# The amount of memory in MB for the lambda that batches up compaction commits.
# (default value shown below, uncomment to set a value)
# sleeper.compaction.commit.batcher.memory.mb=4096

# The timeout for the lambda that batches up compaction commits in seconds.
# (default value shown below, uncomment to set a value)
# sleeper.compaction.commit.batcher.timeout.seconds=900

# The reserved concurrency for the lambda that batches up compaction commits.
# See reserved concurrency overview at:
# https://docs.aws.amazon.com/lambda/latest/dg/configuration-concurrency.html
# (default value shown below, uncomment to set a value)
# sleeper.compaction.commit.batcher.concurrency.reserved=2

# The maximum concurrency allowed for the lambda that batches up compaction commits.
# See maximum concurrency overview at:
# https://aws.amazon.com/blogs/compute/introducing-maximum-concurrency-of-aws-lambda-functions-when-using-amazon-sqs-as-an-event-source/
# (default value shown below, uncomment to set a value)
# sleeper.compaction.commit.batcher.concurrency.max=2

# The maximum number of concurrent compaction tasks to run.
# (default value shown below, uncomment to set a value)
# sleeper.compaction.max.concurrent.tasks=300

# The rate at which a check to see if compaction ECS tasks need to be created is made (in minutes,
# must be >= 1).
# (default value shown below, uncomment to set a value)
# sleeper.compaction.task.creation.period.minutes=1

# The maximum number of times that a compaction job can be taken off the job definition queue before
# it is moved to the dead letter queue.
# This property is used to configure the maxReceiveCount of the compaction job definition queue.
# (default value shown below, uncomment to set a value)
# sleeper.compaction.job.max.retries=3

# The maximum number of times that a batch of compaction jobs can be taken off the pending queue
# before it is moved to the dead letter queue.
# This property is used to configure the maxReceiveCount of the pending compaction job batch queue.
# (default value shown below, uncomment to set a value)
# sleeper.compaction.job.dispatch.max.retries=3

# The maximum number of times that a compaction job can be taken off the batch committer queue before
# it is moved to the dead letter queue.
# This property is used to configure the maxReceiveCount of the compaction job committer queue.
# (default value shown below, uncomment to set a value)
# sleeper.compaction.job.commit.max.retries=3

# The CPU architecture to run compaction tasks on. Valid values are X86_64 and ARM64.
# See Task CPU architecture at
# https://docs.aws.amazon.com/AmazonECS/latest/developerguide/AWS_Fargate.html
# When `sleeper.compaction.ecs.launch.type` is set to EC2, this must match the architecture of the EC2
# type set in `sleeper.compaction.ec2.type`.
# (default value shown below, uncomment to set a value)
# sleeper.compaction.task.cpu.architecture=ARM64

# The CPU for a compaction task using an ARM64 architecture.
# See https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html for valid
# options.
# (default value shown below, uncomment to set a value)
# sleeper.compaction.task.arm.cpu=4096

# The amount of memory in MB for a compaction task using an ARM64 architecture.
# See https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html for valid
# options.
# (default value shown below, uncomment to set a value)
# sleeper.compaction.task.arm.memory.mb=8192

# The CPU for a compaction task using an x86_64 architecture.
# See https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html for valid
# options.
# (default value shown below, uncomment to set a value)
# sleeper.compaction.task.x86.cpu=4096

# The amount of memory in MB for a compaction task using an x86_64 architecture.
# See https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html for valid
# options.
# (default value shown below, uncomment to set a value)
# sleeper.compaction.task.x86.memory.mb=8192

# Used when scaling EC2 instances to support an expected number of compaction tasks. This is the
# amount of memory in MB that we expect ECS to reserve on an EC2 instance before making memory
# available for compaction tasks.
# If this is unset, it will be computed as a percentage of the memory on the EC2 instead, see
# `sleeper.compaction.task.scaling.overhead.percentage`.
# (uncomment to set a value)
# sleeper.compaction.task.scaling.overhead.fixed=

# Used when scaling EC2 instances to support an expected number of compaction tasks. This is the
# percentage of memory in an EC2 instance that we expect ECS to reserve before making memory available
# for compaction tasks.
# Defaults to 10%, so we expect 90% of the memory on an EC2 instance to be used for compaction tasks.
# (default value shown below, uncomment to set a value)
# sleeper.compaction.task.scaling.overhead.percentage=10

# What launch type should compaction containers use? Valid options: FARGATE, EC2.
# (default value shown below, uncomment to set a value)
# sleeper.compaction.ecs.launch.type=FARGATE

# The EC2 instance type to use for compaction tasks (when using EC2-based compactions). Note that the
# architecture configured in `sleeper.compaction.task.cpu.architecture` must match.
# (default value shown below, uncomment to set a value)
# sleeper.compaction.ec2.type=t4g.xlarge

# The minimum number of instances for the EC2 cluster (when using EC2-based compactions).
# (default value shown below, uncomment to set a value)
# sleeper.compaction.ec2.pool.minimum=0

# The maximum number of instances for the EC2 cluster (when using EC2-based compactions).
# (default value shown below, uncomment to set a value)
# sleeper.compaction.ec2.pool.maximum=75

# The size in GiB of the root EBS volume attached to the EC2 instances (when using EC2-based
# compactions).
# (default value shown below, uncomment to set a value)
# sleeper.compaction.ec2.root.size=50

# Flag to enable/disable storage of tracking information for compaction jobs and tasks.
# (default value shown below, uncomment to set a value)
# sleeper.compaction.tracker.enabled=true

# Flag to enable/disable storing an update to the tracker during async commits of compaction jobs.
# This may be disabled if there are enough compactions that the system is unable to apply all the
# updates to the tracker. This is mainly used for testing. Reports may show compactions as unfinished
# if this update is not present in the tracker.
# (default value shown below, uncomment to set a value)
# sleeper.compaction.tracker.async.commit.updates.enabled=true

# The time to live in seconds for compaction job updates in the job tracker. Default is 1 week.
# The expiry time is fixed when an update is saved to the store, so changing this will only affect new
# data.
# (default value shown below, uncomment to set a value)
# sleeper.compaction.job.status.ttl=604800

# The time to live in seconds for compaction task updates in the job tracker. Default is 1 week.
# The expiry time is fixed when an update is saved to the store, so changing this will only affect new
# data.
# (default value shown below, uncomment to set a value)
# sleeper.compaction.task.status.ttl=604800

# The name of the class that defines how compaction jobs should be created. This should implement
# sleeper.compaction.core.job.creation.strategy.CompactionStrategy. The value of this property is the
# default value which can be overridden on a per-table basis.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.compaction.strategy.class=sleeper.compaction.core.job.creation.strategy.impl.SizeRatioCompactionStrategy

# The maximum number of files to read in a compaction job. Note that the state store must support
# atomic updates for this many files.
# Also note that this many files may need to be open simultaneously. The value of
# 'sleeper.fs.s3a.max-connections' must be at least the value of this plus one. The extra one is for
# the output file.
# This is a default value and will be used if not specified in the table properties.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.compaction.files.batch.size=12

# The number of compaction jobs to send in a single batch.
# When compaction jobs are created, there is no limit on how many jobs can be created at once. A batch
# is a group of compaction jobs that will have their creation updates applied at the same time. For
# each batch, we send all compaction jobs to the SQS queue, then update the state store to assign job
# IDs to the input files.
# This can be overridden on a per-table basis.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.compaction.job.send.batch.size=1000

# The amount of time in seconds a batch of compaction jobs may be pending before it should not be
# retried. If the input files have not been successfully assigned to the jobs, and this much time has
# passed, then the batch will fail to send.
# Once a pending batch fails the input files will never be compacted again without other intervention,
# so it's important to ensure file assignment will be done within this time. That depends on the
# throughput of state store commits.
# It's also necessary to ensure file assignment will be done before the next invocation of compaction
# job creation, otherwise invalid jobs will be created for the same input files. The rate of these
# invocations is set in `sleeper.compaction.job.creation.period.minutes`.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.compaction.job.send.timeout.seconds=90

# The amount of time in seconds to wait between attempts to send a batch of compaction jobs. The batch
# will be sent if all input files have been successfully assigned to the jobs, otherwise the batch
# will be retried after a delay.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.compaction.job.send.retry.delay.seconds=30

# The default limit on the number of compactation jobs that can be created within a single
# invocation.Exceeding this limit, results in the selection being randomised.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.compaction.job.creation.limit=100000

# Used by the SizeRatioCompactionStrategy to decide if a group of files should be compacted.
# If the file sizes are s_1, ..., s_n then the files are compacted if s_1 + ... + s_{n-1} >= ratio *
# s_n.
# It can be overridden on a per-table basis.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.compaction.strategy.sizeratio.ratio=3

# Used by the SizeRatioCompactionStrategy to control the maximum number of jobs that can be running
# concurrently per partition. It can be overridden on a per-table basis.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.compaction.strategy.sizeratio.max.concurrent.jobs.per.partition=2147483647


## The following instance properties relate to queries.

# The maximum number of simultaneous connections to S3 from a single query runner. This is separated
# from the main one as it's common for a query runner to need to open more files at once.
# (default value shown below, uncomment to set a value)
# sleeper.query.s3.max-connections=1024

# The amount of memory in MB for the lambda that executes queries.
# (default value shown below, uncomment to set a value)
# sleeper.query.processor.memory.mb=4096

# The timeout for the lambda that executes queries in seconds.
# (default value shown below, uncomment to set a value)
# sleeper.query.processor.timeout.seconds=900

# The frequency with which the query processing lambda refreshes its knowledge of the system state
# (i.e. the partitions and the mapping from partition to files), in seconds.
# (default value shown below, uncomment to set a value)
# sleeper.query.processor.state.refresh.period.seconds=60

# The maximum number of rows to include in a batch of query results send to the results queue from the
# query processing lambda.
# (default value shown below, uncomment to set a value)
# sleeper.query.processor.results.batch.size=2000

# The size of the thread pool for retrieving rows in a query processing lambda.
# (default value shown below, uncomment to set a value)
# sleeper.query.processor.row.retrieval.threads=10

# The default amount of time in seconds the query executor's cache of partition and file reference
# information is valid for. After this it will time out and need refreshing.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.query.processor.cache.timeout.seconds=60

# This value is used to set the time-to-live on the tracking of the queries in the DynamoDB-based
# query tracker.
# (default value shown below, uncomment to set a value)
# sleeper.query.tracker.ttl.days=1

# The length of time the results of queries remain in the query results bucket before being deleted.
# (default value shown below, uncomment to set a value)
# sleeper.query.results.bucket.expiry.days=7

# The visibility timeout in seconds of the query results queue.
# (default value shown below, uncomment to set a value)
# sleeper.query.results.queue.visibility.timeout.seconds=900

# The default value of the rowgroup size used when the results of queries are written to Parquet
# files. The value given below is 8MiB. This value can be overridden using the query config.
# (default value shown below, uncomment to set a value)
# sleeper.default.query.results.rowgroup.size=8388608

# The default value of the page size used when the results of queries are written to Parquet files.
# The value given below is 128KiB. This value can be overridden using the query config.
# (default value shown below, uncomment to set a value)
# sleeper.default.query.results.page.size=131072

# The rate at which the query lambda runs to keep it warm (in minutes, must be >=1).  This only
# applies when the KeepLambdaWarmStack is enabled
# (default value shown below, uncomment to set a value)
# sleeper.query.warm.lambda.period.minutes=5


## The following instance properties relate to metrics.

# The CloudWatch namespace to publish table metrics to.
# (default value shown below, uncomment to set a value)
# sleeper.cloudwatch.table.metrics.namespace=Sleeper

# The reserved concurrency for the table metrics lambda.
# See reserved concurrency overview at:
# https://docs.aws.amazon.com/lambda/latest/dg/configuration-concurrency.html
# (uncomment to set a value)
# sleeper.lambda.table.metrics.concurrency.reserved=

# The maximum concurrency allowed for the table metrics lambda.
# See maximum concurrency overview at:
# https://aws.amazon.com/blogs/compute/introducing-maximum-concurrency-of-aws-lambda-functions-when-using-amazon-sqs-as-an-event-source/
# (default value shown below, uncomment to set a value)
# sleeper.lambda.table.metrics.concurrency.max=10

# The number of tables to calculate metrics for in a single invocation. This will be the batch size
# for a lambda as an SQS FIFO event source. This can be a maximum of 10.
# (default value shown below, uncomment to set a value)
# sleeper.run.table.metrics.batch.size=1

# Whether to calculate table metrics for offline tables.
# (default value shown below, uncomment to set a value)
# sleeper.run.table.metrics.offline=false

# The period in minutes used in the dashboard.
# (default value shown below, uncomment to set a value)
# sleeper.dashboard.time.window.minutes=5

# Enables/disables event count metrics specific to the transaction log follower lambda.
# (default value shown below, uncomment to set a value)
# sleeper.statestore.transactionlog.eventcount.metrics.enabled=false


## The following instance properties relate to logging.

# The logging level for logging Sleeper classes. This does not apply to the MetricsLogger which is
# always set to INFO.
# (uncomment to set a value)
# sleeper.logging.level=

# The logging level for Apache logs that are not Parquet.
# (uncomment to set a value)
# sleeper.logging.apache.level=

# The logging level for Parquet logs.
# (uncomment to set a value)
# sleeper.logging.parquet.level=

# The logging level for AWS logs.
# (uncomment to set a value)
# sleeper.logging.aws.level=

# The logging level for everything else.
# (uncomment to set a value)
# sleeper.logging.root.level=

# Configuration for Rust backtrace generation, set in the environment variable RUST_BACKTRACE.
# (uncomment to set a value)
# sleeper.logging.backtrace=

# Configuration for Rust logging, set in the environment variable RUST_LOG.
# (uncomment to set a value)
# sleeper.logging.rust=


## The following instance properties relate to the integration with Athena.

# The number of days before objects in the spill bucket are deleted.
# (default value shown below, uncomment to set a value)
# sleeper.athena.spill.bucket.ageoff.days=1

# The fully qualified composite classes to deploy. These are the classes that interact with Athena.
# (default value shown below, uncomment to set a value)
# sleeper.athena.handler.classes=sleeper.athena.composite.DataFusionCompositeHandler,sleeper.athena.composite.IteratorApplyingCompositeHandler

# The amount of memory (MB) the athena composite handler has.
# (default value shown below, uncomment to set a value)
# sleeper.athena.handler.memory.mb=4096

# The timeout in seconds for the athena composite handler.
# (default value shown below, uncomment to set a value)
# sleeper.athena.handler.timeout.seconds=900

# ARN of the KMS Key used to encrypt data in the Athena spill bucket.
# (uncomment to set a value)
# sleeper.athena.spill.master.key.arn=


## The following instance properties relate to default values used by table properties.

# Used during Sleeper queries to determine whether the column/offset indexes (also known as page
# indexes) are read from Parquet files. For some queries, e.g. single/few row lookups this can improve
# performance by enabling more aggressive pruning. On range queries, especially on large tables this
# can harm performance, since readers will read the extra index data before returning results, but
# with little benefit from pruning.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.query.parquet.column.index.enabled=false

# Maximum number of bytes to write in a Parquet row group (default is 8MiB). This property is NOT used
# by DataFusion data engine.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.parquet.rowgroup.size=8388608

# The size of the pages in the Parquet files (default is 128KiB).
# (default value shown below, uncomment to set a value)
# sleeper.default.table.parquet.page.size=131072

# The compression codec to use in the Parquet files.
# Valid values are: [uncompressed, snappy, gzip, lzo, brotli, lz4, zstd]
# (default value shown below, uncomment to set a value)
# sleeper.default.table.parquet.compression.codec=zstd

# Whether dictionary encoding should be used for row key columns in the Parquet files.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.parquet.dictionary.encoding.rowkey.fields=false

# Whether dictionary encoding should be used for sort key columns in the Parquet files.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.parquet.dictionary.encoding.sortkey.fields=false

# Whether dictionary encoding should be used for value columns in the Parquet files.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.parquet.dictionary.encoding.value.fields=false

# Used to set parquet.columnindex.truncate.length, see documentation here:
# https://github.com/apache/parquet-mr/blob/master/parquet-hadoop/README.md
# The length in bytes to truncate binary values in a column index.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.parquet.columnindex.truncate.length=128

# Used to set parquet.statistics.truncate.length, see documentation here:
# https://github.com/apache/parquet-mr/blob/master/parquet-hadoop/README.md
# The length in bytes to truncate the min/max binary values in row groups.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.parquet.statistics.truncate.length=2147483647

# Enables a cache of data when reading from S3 with the DataFusion data engine, to hold data in larger
# blocks than are requested by DataFusion.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.datafusion.s3.readahead.enabled=true

# Used to set parquet.writer.version, see documentation here:
# https://github.com/apache/parquet-mr/blob/master/parquet-hadoop/README.md
# Can be either v1 or v2. The v2 pages store levels uncompressed while v1 pages compress levels with
# the data.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.parquet.writer.version=v2

# Maximum number of rows to write in a Parquet row group.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.parquet.rowgroup.rows.max=100000

# The number of attempts to make when applying a transaction to the state store. This default can be
# overridden by a table property.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.statestore.transactionlog.add.transaction.max.attempts=10

# The maximum amount of time to wait before the first retry when applying a transaction to the state
# store. Full jitter will be applied so that the actual wait time will be a random period between 0
# and this value. This ceiling will increase exponentially on further retries. See the below article.
# https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/
# This default can be overridden by a table property.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.statestore.transactionlog.add.transaction.first.retry.wait.ceiling.ms=200

# The maximum amount of time to wait before any retry when applying a transaction to the state store.
# Full jitter will be applied so that the actual wait time will be a random period between 0 and this
# value. This restricts the exponential increase of the wait ceiling while retrying the transaction.
# See the below article.
# https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/
# This default can be overridden by a table property.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.statestore.transactionlog.add.transaction.max.retry.wait.ceiling.ms=30000

# The number of elements to include per Arrow row batch in a snapshot derived from the transaction
# log, of the state of files in a Sleeper table. Each file includes some number of references on
# different partitions. Each reference will count for one element in a row batch, but a file cannot
# currently be split between row batches. A row batch may contain more file references than this if a
# single file overflows the batch. A file with no references counts as one element.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.statestore.transactionlog.files.snapshot.batch.size=1000

# The number of partitions to include per Arrow row batch in a snapshot derived from the transaction
# log, of the state of partitions in a Sleeper table.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.statestore.transactionlog.partitions.snapshot.batch.size=1000

# The number of seconds to wait after we've loaded a snapshot before looking for a new snapshot. This
# should relate to the rate at which new snapshots are created, configured in the instance property
# `sleeper.statestore.transactionlog.snapshot.creation.lambda.period.seconds`. This default can be
# overridden by a table property.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.statestore.transactionlog.time.between.snapshot.checks.seconds=60

# The number of milliseconds to wait after we've updated from the transaction log before checking for
# new transactions. The state visible to an instance of the state store can be out of date by this
# amount. This can avoid excessive queries by the same process, but can result in unwanted behaviour
# when using multiple state store objects. When adding a new transaction to update the state, this
# will be ignored and the state will be brought completely up to date. This default can be overridden
# by a table property.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.statestore.transactionlog.time.between.transaction.checks.ms=0

# The minimum number of transactions that a snapshot must be ahead of the local state, before we load
# the snapshot instead of updating from the transaction log.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.statestore.transactionlog.snapshot.load.min.transactions.ahead=10

# The number of days that transaction log snapshots remain in the snapshot store before being deleted.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.statestore.transactionlog.snapshot.expiry.days=2

# The minimum age in minutes of a snapshot in order to allow deletion of transactions leading up to
# it. When deleting old transactions, there's a chance that processes may still read transactions
# starting from an older snapshot. We need to avoid deletion of any transactions associated with a
# snapshot that may still be used as the starting point for reading the log.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.statestore.transactionlog.delete.behind.snapshot.min.age.minutes=2

# The minimum number of transactions that a transaction must be behind the latest snapshot before
# being deleted. This is the number of transactions that will be kept and protected from deletion,
# whenever old transactions are deleted. This includes the transaction that the latest snapshot was
# created against. Any transactions after the snapshot will never be deleted as they are still in
# active use.
# This should be configured in relation to the property which determines whether a process will load
# the latest snapshot or instead seek through the transaction log, since we need to preserve
# transactions that may still be read:
# sleeper.default.statestore.snapshot.load.min.transactions.ahead
# The snapshot that will be considered the latest snapshot is configured by a property to set the
# minimum age for it to count for this:
# sleeper.default.statestore.transactionlog.delete.behind.snapshot.min.age
# (default value shown below, uncomment to set a value)
# sleeper.default.table.statestore.transactionlog.delete.number.behind.latest.snapshot=200

# Specifies the minimum number of leaf partitions that are needed to run a bulk import job. If this
# minimum has not been reached, bulk import jobs will refuse to start.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.bulk.import.min.leaf.partitions=256

# Specifies the number of times bulk import tries to create leaf partitions to meet the minimum number
# of leaf partitions. This will be retried if another process splits the same partitions at the same
# time.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.bulk.import.partition.splitting.attempts=3

# Specifies the minimum total file size required for an ingest job to be batched and sent. An ingest
# job will be created if the batcher runs while this much data is waiting, and the minimum number of
# files is also met.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.ingest.batcher.job.min.size=1G

# Specifies the maximum total file size for a job in the ingest batcher. If more data is waiting than
# this, it will be split into multiple jobs. If a single file exceeds this, it will still be ingested
# in its own job. It's also possible some data may be left for a future run of the batcher if some
# recent files overflow the size of a job but aren't enough to create a job on their own.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.ingest.batcher.job.max.size=5G

# Specifies the minimum number of files for a job in the ingest batcher. An ingest job will be created
# if the batcher runs while this many files are waiting, and the minimum size of files is also met.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.ingest.batcher.job.min.files=1

# Specifies the maximum number of files for a job in the ingest batcher. If more files are waiting
# than this, they will be split into multiple jobs. It's possible some data may be left for a future
# run of the batcher if some recent files overflow the size of a job but aren't enough to create a job
# on their own.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.ingest.batcher.job.max.files=100

# Specifies the maximum time in seconds that a file can be held in the batcher before it will be
# included in an ingest job. When any file has been waiting for longer than this, jobs will be created
# for all the currently held files, even if other criteria for a batch are not met.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.ingest.batcher.file.max.age.seconds=300

# Specifies the target ingest queue where batched jobs are sent.
# Valid values are: [standard_ingest, bulk_import_emr, bulk_import_persistent_emr, bulk_import_eks,
# bulk_import_emr_serverless]
# (default value shown below, uncomment to set a value)
# sleeper.default.table.ingest.batcher.ingest.queue=bulk_import_emr_serverless

# The time in minutes that the tracking information is retained for a file before the records of its
# ingest are deleted (eg. which ingest job it was assigned to, the time this occurred, the size of the
# file).
# The expiry time is fixed when a file is saved to the store, so changing this will only affect new
# data.
# Defaults to 1 week.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.ingest.batcher.file.tracking.ttl.minutes=10080

# Specifies the strategy that ingest uses to create files and references in partitions.
# Valid values are: [one_file_per_leaf, one_reference_per_leaf]
# (default value shown below, uncomment to set a value)
# sleeper.default.table.ingest.file.writing.strategy=one_reference_per_leaf

# The way in which rows are held in memory before they are written to a local store.
# Valid values are 'arraylist' and 'arrow'.
# The arraylist method is simpler, but it is slower and requires careful tuning of the number of rows
# in each batch.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.ingest.row.batch.type=arrow

# The way in which partition files are written to the main Sleeper store.
# Valid values are 'direct' (which writes using the s3a Hadoop file system) and 'async' (which writes
# locally and then copies the completed Parquet file asynchronously into S3).
# The direct method is simpler but the async method should provide better performance when the number
# of partitions is large.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.ingest.partition.file.writer.type=async

# This is the default for whether state store updates will be applied asynchronously via the state
# store committer.
# This is usually only used for state store implementations where there's a benefit to applying state
# store updates in a single process for each Sleeper table. This is usually to avoid contention from
# multiple processes performing updates at the same time.
# This is separate from the properties that determine which state store updates will be done as
# asynchronous commits. Those properties will only be applied when asynchronous commits are enabled
# for a given state store.
# Valid values are: [disabled, per_implementation, all_implementations]
# With `disabled`, asynchronous commits will never be used unless overridden in table properties.
# With `per_implementation`, asynchronous commits will be used for all state store implementations
# that are known to benefit from it, unless overridden in table properties.
# With `all_implementations`, asynchronous commits will be used for all state stores unless overridden
# in table properties.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.statestore.commit.async.behaviour=PER_IMPLEMENTATION

# This is the default for whether created compaction jobs will be assigned to their input files
# asynchronously via the state store committer, if asynchronous commit is enabled. Otherwise, the
# compaction job creator will commit input file assignments directly to the state store.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.compaction.job.id.assignment.commit.async=true

# This is the default for whether compaction tasks will commit finished jobs asynchronously via the
# state store committer, if asynchronous commit is enabled. Otherwise, compaction tasks will commit
# finished jobs directly to the state store.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.compaction.job.commit.async=true

# This property is the default for whether commits of compaction jobs are batched before being sent to
# the state store commit queue to be applied by the committer lambda. If this property is true and
# asynchronous commits are enabled then commits of compactions will be batched. If this property is
# false and asynchronous commits are enabled then commits of compactions will not be batched and will
# be sent directly to the committer lambda. This property can be overridden for individual tables.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.compaction.job.async.commit.batching=true

# This is the default for whether ingest tasks will add files asynchronously via the state store
# committer, if asynchronous commit is enabled. Otherwise, ingest tasks will add files directly to the
# state store.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.ingest.job.files.commit.async=true

# This is the default for whether bulk import will add files asynchronously via the state store
# committer, if asynchronous commit is enabled. Otherwise, bulk import will add files directly to the
# state store.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.bulk.import.job.files.commit.async=true

# This is the default for whether partition splits will be applied asynchronously via the state store
# committer, if asynchronous commit is enabled. Otherwise, the partition splitter will apply splits
# directly to the state store.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.partition.splitting.commit.async=true

# This is the default for whether the garbage collector will record deleted files asynchronously via
# the state store committer, if asynchronous commit is enabled. Otherwise, the garbage collector will
# record this directly to the state store.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.gc.commit.async=true

# When using the transaction log state store, this sets whether to update from the transaction log
# before adding a transaction in the asynchronous state store committer.
# If asynchronous commits are used for all or almost all state store updates, this can be false to
# avoid the extra queries.
# If the state store is commonly updated directly outside of the asynchronous committer, this can be
# true to avoid conflicts and retries.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.statestore.committer.update.every.commit=false

# When using the transaction log state store, this sets whether to update from the transaction log
# before adding a batch of transactions in the asynchronous state store committer.
# (default value shown below, uncomment to set a value)
# sleeper.default.table.statestore.committer.update.every.batch=true

# Select which data engine to use for the table. Valid values are: [java, datafusion,
# datafusion_experimental]
# (default value shown below, uncomment to set a value)
# sleeper.default.table.data.engine=DATAFUSION