diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000000000000000000000000000000000000..3744ea759c93e6a63e473e0de70f5798393037e3 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,18 @@ +default: + image: docker:26.0.1 + services: + - docker:26.0.1-dind + before_script: + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + interruptible: true + +stages: + - build + +s5cmd-partition-build: + stage: build + script: + - docker pull $CI_REGISTRY_IMAGE/s5cmd_dask:latest || true + - docker build --cache-from $CI_REGISTRY_IMAGE/s5cmd_dask:latest --tag $CI_REGISTRY_IMAGE/s5cmd_dask:$CI_COMMIT_SHA --tag $CI_REGISTRY_IMAGE/s5cmd_dask:latest -f prep-parquet-for-s5cmd/Dockerfile . + - docker push $CI_REGISTRY_IMAGE/s5cmd_dask:$CI_COMMIT_SHA + - docker push $CI_REGISTRY_IMAGE/s5cmd_dask:latest diff --git a/prep-parquet-for-s5cmd/Dockerfile b/prep-parquet-for-s5cmd/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..0a9d665ab4590c1dd1e601829ded815ebc816bc5 --- /dev/null +++ b/prep-parquet-for-s5cmd/Dockerfile @@ -0,0 +1,3 @@ +FROM daskdev/dask:2024.8.0-py3.11 + +RUN conda install conda-forge::s5cmd \ No newline at end of file