From fde6d8ec9a56ffe1bffc29ace96599844f1d4950 Mon Sep 17 00:00:00 2001 From: Alexander Shires Date: Thu, 16 Feb 2023 09:53:58 +0000 Subject: [PATCH 01/28] adding git diff utilitty --- .../cicd_sample/utils/add_dags_to_composer.py | 4 +- .../utils/deploy_dags_from_diff.py | 93 +++++++++++++++++++ 2 files changed, 95 insertions(+), 2 deletions(-) create mode 100644 composer/cicd_sample/utils/deploy_dags_from_diff.py diff --git a/composer/cicd_sample/utils/add_dags_to_composer.py b/composer/cicd_sample/utils/add_dags_to_composer.py index 82023e4df38..5b3ba92ad9a 100644 --- a/composer/cicd_sample/utils/add_dags_to_composer.py +++ b/composer/cicd_sample/utils/add_dags_to_composer.py @@ -40,7 +40,7 @@ def _create_dags_list(dags_directory: str) -> Tuple[str, List[str]]: def upload_dags_to_composer( - dags_directory: str, bucket_name: str, name_replacement: str = "dags/" + dags_directory: str, bucket_name: str, name_replacement: str = "dags/" ) -> None: """ Given a directory, this function moves all DAG files from that directory @@ -97,4 +97,4 @@ def upload_dags_to_composer( args = parser.parse_args() upload_dags_to_composer(args.dags_directory, args.dags_bucket) -# [END composer_cicd_add_dags_to_composer_utility] +# [END composer_cicd_add_dags_to_composer_utility] \ No newline at end of file diff --git a/composer/cicd_sample/utils/deploy_dags_from_diff.py b/composer/cicd_sample/utils/deploy_dags_from_diff.py new file mode 100644 index 00000000000..42a58689891 --- /dev/null +++ b/composer/cicd_sample/utils/deploy_dags_from_diff.py @@ -0,0 +1,93 @@ +# Copyright 2021 Google LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# https://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START composer_cicd_deploy_dags_from_diff_utility] + +import argparse +import glob +import os +import git +from pathlib import Path +from shutil import copytree, ignore_patterns +import tempfile +from typing import List, Tuple + +# Imports the Google Cloud client library +from google.cloud import storage + + +def _create_dags_list_from_git_diff(dag_dir: str, repo_root: str, main_branch_name: str) -> Tuple[str, List[Path]]: + """ + get the list of files within the DAG dir that have changed in the latest git commits against the specified branch + :param dag_dir: + :param repo_root: + :param main_branch_name: + :return: + """ + repo = git.Repo(repo_root) + diff_results = repo.git.diff(main_branch_name) + p = Path(repo_root) + changed_file_list = list() + for diff_result_line in diff_results.split("\n"): + # test if changed + if "+++ b" in diff_result_line and dag_dir in diff_result_line: + changed_path = p / Path(diff_result_line[6:]) + changed_file_list.append(changed_path) + # print(changed_path.resolve()) + return dag_dir, changed_file_list + + +def upload_changed_dags_to_composer(dag_list: List[Path], dag_bucket: str) -> None: + """ + list of DAGs to upload to Commposer + :param dag_list: + :return: + """ + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + + + + + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--dags_directory", + help="Relative path to the source directory containing your DAGs", + ) + parser.add_argument( + "--dags_bucket", + help="Name of the DAGs bucket of your Composer environment without the gs:// prefix", + ) + parser.add_argument( + "--dag_repo", + required=False, + help="Relative path to the root of the git repo containing the DAGs", + ) + parser.add_argument( + "--repo_main", + required=False, + help="Main branch of the DAG git repo to compare changes against", + ) + + args = parser.parse_args() + + dag_list = _create_dags_list_from_git_diff(args.dags_directory, args.repo_root, args.repo_main) + + upload_changed_dags_to_composer(dag_list, args.dags_bucket) +# [END composer_cicd_deploy_dags_from_diff_utility] \ No newline at end of file From ad7edc00e946def4198386a61d3502fed83d2328 Mon Sep 17 00:00:00 2001 From: Alexander Shires Date: Thu, 16 Feb 2023 10:42:38 +0000 Subject: [PATCH 02/28] adding tests --- .../utils/deploy_dags_from_diff.py | 21 ++++--- .../utils/deploy_dags_from_diff_test.py | 57 +++++++++++++++++++ 2 files changed, 69 insertions(+), 9 deletions(-) create mode 100644 composer/cicd_sample/utils/deploy_dags_from_diff_test.py diff --git a/composer/cicd_sample/utils/deploy_dags_from_diff.py b/composer/cicd_sample/utils/deploy_dags_from_diff.py index 42a58689891..0b7dea28d25 100644 --- a/composer/cicd_sample/utils/deploy_dags_from_diff.py +++ b/composer/cicd_sample/utils/deploy_dags_from_diff.py @@ -1,4 +1,4 @@ -# Copyright 2021 Google LLC +# Copyright 2023 Google LLC # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -27,7 +27,7 @@ from google.cloud import storage -def _create_dags_list_from_git_diff(dag_dir: str, repo_root: str, main_branch_name: str) -> Tuple[str, List[Path]]: +def create_dags_list_from_git_diff(dag_dir: str, repo_root: str, main_branch_name: str) -> Tuple[str, List[Path]]: """ get the list of files within the DAG dir that have changed in the latest git commits against the specified branch :param dag_dir: @@ -48,7 +48,7 @@ def _create_dags_list_from_git_diff(dag_dir: str, repo_root: str, main_branch_na return dag_dir, changed_file_list -def upload_changed_dags_to_composer(dag_list: List[Path], dag_bucket: str) -> None: +def upload_changed_dags_to_composer(dag_list: List[Path], bucket_name: str) -> None: """ list of DAGs to upload to Commposer :param dag_list: @@ -56,10 +56,10 @@ def upload_changed_dags_to_composer(dag_list: List[Path], dag_bucket: str) -> No """ storage_client = storage.Client() bucket = storage_client.bucket(bucket_name) - - - - + for dag_path in dag_list: + blob = bucket.blob(dag_path.name) + blob.upload_from_filename(str(dag_path)) + print(f"File {dag_path.name} uploaded to {bucket_name}/{dag_path.name}.") if __name__ == "__main__": @@ -87,7 +87,10 @@ def upload_changed_dags_to_composer(dag_list: List[Path], dag_bucket: str) -> No args = parser.parse_args() - dag_list = _create_dags_list_from_git_diff(args.dags_directory, args.repo_root, args.repo_main) + dag_list = create_dags_list_from_git_diff(args.dags_directory, args.repo_root, args.repo_main) - upload_changed_dags_to_composer(dag_list, args.dags_bucket) + if len(dag_list)==0: + print("No DAGs to upload") + else: + upload_changed_dags_to_composer(dag_list=dag_list, bucket_name=args.dags_bucket) # [END composer_cicd_deploy_dags_from_diff_utility] \ No newline at end of file diff --git a/composer/cicd_sample/utils/deploy_dags_from_diff_test.py b/composer/cicd_sample/utils/deploy_dags_from_diff_test.py new file mode 100644 index 00000000000..ddf526844e1 --- /dev/null +++ b/composer/cicd_sample/utils/deploy_dags_from_diff_test.py @@ -0,0 +1,57 @@ +# Copyright 2023 Google LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# https://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import pathlib +from shutil import copytree +import tempfile +import uuid +import git + +from google.cloud import storage +import pytest + +import deploy_dags_from_diff # noqa: I100 - lint is incorrectly saying this is out of order + +DAGS_DIR = pathlib.Path(__file__).parent.parent / "dags/" +REPO_ROOT = pathlib.Path(__file__).parent.parent.parent.parent +REPO_MAIN = "main" + + +def test_create_dags_list_with_changes() -> None: + """ + this test checks for + :return: + """ + repo = git.Repo(REPO_ROOT) + + with open(DAGS_DIR+"/example2_dag.py", 'w') as f: + # create minor + + dag_list = deploy_dags_from_diff.create_dags_list_from_git_diff(DAGS_DIR, REPO_ROOT, REPO_MAIN) + assert len(dag_list) > 0 + assert + + + + +def test_create_dags_list_no_changes() -> None: + """ + this test checks for an empty list based on no changes in git + :return: + """ + dag_list = deploy_dags_from_diff.create_dags_list_from_git_diff(DAGS_DIR, REPO_ROOT, REPO_MAIN) + assert len(dag_list) == 0 + From c8fb9bcfc68064cbdc2eb60a5ae62db3ecf1ab91 Mon Sep 17 00:00:00 2001 From: Alexander Shires Date: Thu, 16 Feb 2023 10:53:57 +0000 Subject: [PATCH 03/28] appended line to file From 5d383b4e571f6a0871322135c06f7c5966d9b94a Mon Sep 17 00:00:00 2001 From: Alexander Shires Date: Thu, 16 Feb 2023 10:56:03 +0000 Subject: [PATCH 04/28] appended line to file From 794a0cb0512c5ba52a5d136a210847b4d7a31b6c Mon Sep 17 00:00:00 2001 From: Alexander Shires Date: Thu, 16 Feb 2023 10:56:29 +0000 Subject: [PATCH 05/28] appended line to file From 0d364196513384d083bf1ef742328a9915ea2566 Mon Sep 17 00:00:00 2001 From: Alexander Shires Date: Thu, 16 Feb 2023 10:56:54 +0000 Subject: [PATCH 06/28] appended line to file From 92a599b7c1fa1d1b615fc0deff473792799d43b8 Mon Sep 17 00:00:00 2001 From: Alexander Shires Date: Thu, 16 Feb 2023 10:59:05 +0000 Subject: [PATCH 07/28] appended line to file From 80be4f3d8595c4f8d289194fa8e9cb159f93b75e Mon Sep 17 00:00:00 2001 From: Alexander Shires Date: Thu, 16 Feb 2023 10:59:16 +0000 Subject: [PATCH 08/28] appended line to file From 4247ef815cb9b904b2f51bb1c6756557b7871082 Mon Sep 17 00:00:00 2001 From: Alexander Shires Date: Thu, 16 Feb 2023 10:59:33 +0000 Subject: [PATCH 09/28] appended line to file From 537fe93aac7d0fe6630bb8cee33303aa8acf4469 Mon Sep 17 00:00:00 2001 From: Alexander Shires Date: Thu, 16 Feb 2023 11:00:20 +0000 Subject: [PATCH 10/28] appended line to file From 8a4723e240d22c851f671a97ba2dbe512566fefd Mon Sep 17 00:00:00 2001 From: Alexander Shires Date: Thu, 16 Feb 2023 11:00:40 +0000 Subject: [PATCH 11/28] appended line to file From 15de45afdd362ca65b705b189c264611c2dc07f7 Mon Sep 17 00:00:00 2001 From: Alexander Shires Date: Thu, 16 Feb 2023 11:01:05 +0000 Subject: [PATCH 12/28] appended line to file From 93a2bd424321096f06075e1e7031a41c15bb2ee6 Mon Sep 17 00:00:00 2001 From: Alexander Shires Date: Thu, 16 Feb 2023 11:02:18 +0000 Subject: [PATCH 13/28] appended line to file From e7ef78fbd9551238bec79d7b2348e661572afe42 Mon Sep 17 00:00:00 2001 From: Alexander Shires Date: Thu, 16 Feb 2023 11:04:50 +0000 Subject: [PATCH 14/28] appended line to file --- composer/cicd_sample/dags/example2_dag.py | 1 + .../utils/deploy_dags_from_diff.py | 6 +++-- .../utils/deploy_dags_from_diff_test.py | 24 ++++++++++++------- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/composer/cicd_sample/dags/example2_dag.py b/composer/cicd_sample/dags/example2_dag.py index c050447e330..f2e5b104c41 100644 --- a/composer/cicd_sample/dags/example2_dag.py +++ b/composer/cicd_sample/dags/example2_dag.py @@ -44,3 +44,4 @@ print_dag_run_conf = bash.BashOperator( task_id="print_dag_run_conf", bash_command="echo {{ dag_run.id }}" ) +# appended line# appended line# appended line# appended line \ No newline at end of file diff --git a/composer/cicd_sample/utils/deploy_dags_from_diff.py b/composer/cicd_sample/utils/deploy_dags_from_diff.py index 0b7dea28d25..90ea85280a3 100644 --- a/composer/cicd_sample/utils/deploy_dags_from_diff.py +++ b/composer/cicd_sample/utils/deploy_dags_from_diff.py @@ -27,7 +27,7 @@ from google.cloud import storage -def create_dags_list_from_git_diff(dag_dir: str, repo_root: str, main_branch_name: str) -> Tuple[str, List[Path]]: +def create_dags_list_from_git_diff(dag_dir: str, repo_root: str, main_branch_name: str) -> List[Path]: """ get the list of files within the DAG dir that have changed in the latest git commits against the specified branch :param dag_dir: @@ -37,15 +37,17 @@ def create_dags_list_from_git_diff(dag_dir: str, repo_root: str, main_branch_nam """ repo = git.Repo(repo_root) diff_results = repo.git.diff(main_branch_name) + print(diff_results) p = Path(repo_root) changed_file_list = list() for diff_result_line in diff_results.split("\n"): # test if changed if "+++ b" in diff_result_line and dag_dir in diff_result_line: changed_path = p / Path(diff_result_line[6:]) + print(changed_path) changed_file_list.append(changed_path) # print(changed_path.resolve()) - return dag_dir, changed_file_list + return changed_file_list def upload_changed_dags_to_composer(dag_list: List[Path], bucket_name: str) -> None: diff --git a/composer/cicd_sample/utils/deploy_dags_from_diff_test.py b/composer/cicd_sample/utils/deploy_dags_from_diff_test.py index ddf526844e1..565277ada43 100644 --- a/composer/cicd_sample/utils/deploy_dags_from_diff_test.py +++ b/composer/cicd_sample/utils/deploy_dags_from_diff_test.py @@ -28,6 +28,7 @@ DAGS_DIR = pathlib.Path(__file__).parent.parent / "dags/" REPO_ROOT = pathlib.Path(__file__).parent.parent.parent.parent REPO_MAIN = "main" +TEST_DAG_FILENAME = "example2_dag.py" def test_create_dags_list_with_changes() -> None: @@ -35,16 +36,23 @@ def test_create_dags_list_with_changes() -> None: this test checks for :return: """ + full_dag_dir_str = str(DAGS_DIR.resolve()) + full_repo_root_str = str(REPO_ROOT.resolve()) repo = git.Repo(REPO_ROOT) - - with open(DAGS_DIR+"/example2_dag.py", 'w') as f: - # create minor - - dag_list = deploy_dags_from_diff.create_dags_list_from_git_diff(DAGS_DIR, REPO_ROOT, REPO_MAIN) + repo.create_head("test-branch") + repo.git.checkout("test-branch") + with open(f"{DAGS_DIR.resolve()}/{TEST_DAG_FILENAME}", 'a') as f: + # create minor change + f.write("# appended line") + + repo.git.commit("-am", "appended line to file") + dag_list = deploy_dags_from_diff.create_dags_list_from_git_diff( + full_dag_dir_str, full_repo_root_str, REPO_MAIN) + print(dag_list) assert len(dag_list) > 0 - assert - - + assert "example2_dag.py" in dag_list[0] + # cleanup + repo.git.checkout(REPO_MAIN) def test_create_dags_list_no_changes() -> None: From ee67b7e2bbe8855984c15ea499882589c1b8141a Mon Sep 17 00:00:00 2001 From: Alexander Shires Date: Thu, 16 Feb 2023 11:05:07 +0000 Subject: [PATCH 15/28] appended line to file --- composer/cicd_sample/dags/example2_dag.py | 2 +- composer/cicd_sample/utils/deploy_dags_from_diff.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/composer/cicd_sample/dags/example2_dag.py b/composer/cicd_sample/dags/example2_dag.py index f2e5b104c41..f982404dd34 100644 --- a/composer/cicd_sample/dags/example2_dag.py +++ b/composer/cicd_sample/dags/example2_dag.py @@ -44,4 +44,4 @@ print_dag_run_conf = bash.BashOperator( task_id="print_dag_run_conf", bash_command="echo {{ dag_run.id }}" ) -# appended line# appended line# appended line# appended line \ No newline at end of file +# appended line# appended line# appended line# appended line# appended line \ No newline at end of file diff --git a/composer/cicd_sample/utils/deploy_dags_from_diff.py b/composer/cicd_sample/utils/deploy_dags_from_diff.py index 90ea85280a3..e0b47a94a33 100644 --- a/composer/cicd_sample/utils/deploy_dags_from_diff.py +++ b/composer/cicd_sample/utils/deploy_dags_from_diff.py @@ -37,7 +37,7 @@ def create_dags_list_from_git_diff(dag_dir: str, repo_root: str, main_branch_nam """ repo = git.Repo(repo_root) diff_results = repo.git.diff(main_branch_name) - print(diff_results) + # print(diff_results) p = Path(repo_root) changed_file_list = list() for diff_result_line in diff_results.split("\n"): From 977d0d8c9ce108c10fd5020f5f256a6d678fed36 Mon Sep 17 00:00:00 2001 From: Alexander Shires Date: Thu, 16 Feb 2023 11:06:52 +0000 Subject: [PATCH 16/28] appended line to file --- composer/cicd_sample/dags/example2_dag.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer/cicd_sample/dags/example2_dag.py b/composer/cicd_sample/dags/example2_dag.py index f982404dd34..99c31c61328 100644 --- a/composer/cicd_sample/dags/example2_dag.py +++ b/composer/cicd_sample/dags/example2_dag.py @@ -44,4 +44,4 @@ print_dag_run_conf = bash.BashOperator( task_id="print_dag_run_conf", bash_command="echo {{ dag_run.id }}" ) -# appended line# appended line# appended line# appended line# appended line \ No newline at end of file +# appended line# appended line# appended line# appended line# appended line# appended line \ No newline at end of file From d0e378e54497949b390f49573f071fdc6c1362c9 Mon Sep 17 00:00:00 2001 From: Alexander Shires Date: Thu, 16 Feb 2023 11:08:43 +0000 Subject: [PATCH 17/28] appended line to file --- composer/cicd_sample/dags/example2_dag.py | 2 +- composer/cicd_sample/utils/deploy_dags_from_diff_test.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/composer/cicd_sample/dags/example2_dag.py b/composer/cicd_sample/dags/example2_dag.py index 99c31c61328..35733ee3333 100644 --- a/composer/cicd_sample/dags/example2_dag.py +++ b/composer/cicd_sample/dags/example2_dag.py @@ -44,4 +44,4 @@ print_dag_run_conf = bash.BashOperator( task_id="print_dag_run_conf", bash_command="echo {{ dag_run.id }}" ) -# appended line# appended line# appended line# appended line# appended line# appended line \ No newline at end of file +# appended line# appended line# appended line# appended line# appended line# appended line# appended line diff --git a/composer/cicd_sample/utils/deploy_dags_from_diff_test.py b/composer/cicd_sample/utils/deploy_dags_from_diff_test.py index 565277ada43..eb736280566 100644 --- a/composer/cicd_sample/utils/deploy_dags_from_diff_test.py +++ b/composer/cicd_sample/utils/deploy_dags_from_diff_test.py @@ -43,7 +43,7 @@ def test_create_dags_list_with_changes() -> None: repo.git.checkout("test-branch") with open(f"{DAGS_DIR.resolve()}/{TEST_DAG_FILENAME}", 'a') as f: # create minor change - f.write("# appended line") + f.write("# appended line\n") repo.git.commit("-am", "appended line to file") dag_list = deploy_dags_from_diff.create_dags_list_from_git_diff( @@ -51,8 +51,8 @@ def test_create_dags_list_with_changes() -> None: print(dag_list) assert len(dag_list) > 0 assert "example2_dag.py" in dag_list[0] - # cleanup - repo.git.checkout(REPO_MAIN) + # cleanup - uncomment once test dev is done + # repo.git.checkout(REPO_MAIN) def test_create_dags_list_no_changes() -> None: From 982d7b4eae9de4e00befc63f2cc421868012faef Mon Sep 17 00:00:00 2001 From: Alexander Shires Date: Thu, 16 Feb 2023 11:08:56 +0000 Subject: [PATCH 18/28] appended line to file --- composer/cicd_sample/dags/example2_dag.py | 1 + composer/cicd_sample/utils/deploy_dags_from_diff.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/composer/cicd_sample/dags/example2_dag.py b/composer/cicd_sample/dags/example2_dag.py index 35733ee3333..ce15c03bd28 100644 --- a/composer/cicd_sample/dags/example2_dag.py +++ b/composer/cicd_sample/dags/example2_dag.py @@ -45,3 +45,4 @@ task_id="print_dag_run_conf", bash_command="echo {{ dag_run.id }}" ) # appended line# appended line# appended line# appended line# appended line# appended line# appended line +# appended line diff --git a/composer/cicd_sample/utils/deploy_dags_from_diff.py b/composer/cicd_sample/utils/deploy_dags_from_diff.py index e0b47a94a33..90ea85280a3 100644 --- a/composer/cicd_sample/utils/deploy_dags_from_diff.py +++ b/composer/cicd_sample/utils/deploy_dags_from_diff.py @@ -37,7 +37,7 @@ def create_dags_list_from_git_diff(dag_dir: str, repo_root: str, main_branch_nam """ repo = git.Repo(repo_root) diff_results = repo.git.diff(main_branch_name) - # print(diff_results) + print(diff_results) p = Path(repo_root) changed_file_list = list() for diff_result_line in diff_results.split("\n"): From db6bde17372e4e2e00680c7509231a2358ad90d1 Mon Sep 17 00:00:00 2001 From: Alexander Shires Date: Thu, 16 Feb 2023 11:09:53 +0000 Subject: [PATCH 19/28] appended line to file --- composer/cicd_sample/dags/example2_dag.py | 1 + composer/cicd_sample/utils/deploy_dags_from_diff_test.py | 1 + 2 files changed, 2 insertions(+) diff --git a/composer/cicd_sample/dags/example2_dag.py b/composer/cicd_sample/dags/example2_dag.py index ce15c03bd28..69d553e48a9 100644 --- a/composer/cicd_sample/dags/example2_dag.py +++ b/composer/cicd_sample/dags/example2_dag.py @@ -46,3 +46,4 @@ ) # appended line# appended line# appended line# appended line# appended line# appended line# appended line # appended line +# appended line diff --git a/composer/cicd_sample/utils/deploy_dags_from_diff_test.py b/composer/cicd_sample/utils/deploy_dags_from_diff_test.py index eb736280566..7919dc7e4b9 100644 --- a/composer/cicd_sample/utils/deploy_dags_from_diff_test.py +++ b/composer/cicd_sample/utils/deploy_dags_from_diff_test.py @@ -38,6 +38,7 @@ def test_create_dags_list_with_changes() -> None: """ full_dag_dir_str = str(DAGS_DIR.resolve()) full_repo_root_str = str(REPO_ROOT.resolve()) + print(full_dag_dir_str, full_repo_root_str) repo = git.Repo(REPO_ROOT) repo.create_head("test-branch") repo.git.checkout("test-branch") From 01bb94f08c0a60678fb3c4a3014336ef8d8b7908 Mon Sep 17 00:00:00 2001 From: Alexander Shires Date: Thu, 16 Feb 2023 11:10:11 +0000 Subject: [PATCH 20/28] appended line to file --- composer/cicd_sample/dags/example2_dag.py | 1 + composer/cicd_sample/utils/deploy_dags_from_diff_test.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/composer/cicd_sample/dags/example2_dag.py b/composer/cicd_sample/dags/example2_dag.py index 69d553e48a9..c48b871bf29 100644 --- a/composer/cicd_sample/dags/example2_dag.py +++ b/composer/cicd_sample/dags/example2_dag.py @@ -47,3 +47,4 @@ # appended line# appended line# appended line# appended line# appended line# appended line# appended line # appended line # appended line +# appended line diff --git a/composer/cicd_sample/utils/deploy_dags_from_diff_test.py b/composer/cicd_sample/utils/deploy_dags_from_diff_test.py index 7919dc7e4b9..de808c67607 100644 --- a/composer/cicd_sample/utils/deploy_dags_from_diff_test.py +++ b/composer/cicd_sample/utils/deploy_dags_from_diff_test.py @@ -36,7 +36,7 @@ def test_create_dags_list_with_changes() -> None: this test checks for :return: """ - full_dag_dir_str = str(DAGS_DIR.resolve()) + full_dag_dir_str = str(DAGS_DIR) # .resolve()) full_repo_root_str = str(REPO_ROOT.resolve()) print(full_dag_dir_str, full_repo_root_str) repo = git.Repo(REPO_ROOT) From f6037e1fc602bcf73d7afeb90ec19a829601f2f4 Mon Sep 17 00:00:00 2001 From: Alexander Shires Date: Thu, 16 Feb 2023 11:10:27 +0000 Subject: [PATCH 21/28] appended line to file --- composer/cicd_sample/dags/example2_dag.py | 1 + composer/cicd_sample/utils/deploy_dags_from_diff.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/composer/cicd_sample/dags/example2_dag.py b/composer/cicd_sample/dags/example2_dag.py index c48b871bf29..9dc9620202e 100644 --- a/composer/cicd_sample/dags/example2_dag.py +++ b/composer/cicd_sample/dags/example2_dag.py @@ -48,3 +48,4 @@ # appended line # appended line # appended line +# appended line diff --git a/composer/cicd_sample/utils/deploy_dags_from_diff.py b/composer/cicd_sample/utils/deploy_dags_from_diff.py index 90ea85280a3..e0b47a94a33 100644 --- a/composer/cicd_sample/utils/deploy_dags_from_diff.py +++ b/composer/cicd_sample/utils/deploy_dags_from_diff.py @@ -37,7 +37,7 @@ def create_dags_list_from_git_diff(dag_dir: str, repo_root: str, main_branch_nam """ repo = git.Repo(repo_root) diff_results = repo.git.diff(main_branch_name) - print(diff_results) + # print(diff_results) p = Path(repo_root) changed_file_list = list() for diff_result_line in diff_results.split("\n"): From bb578b5d6d1f71271a762824ed27a559d1ca366f Mon Sep 17 00:00:00 2001 From: Alexander Shires Date: Thu, 16 Feb 2023 11:11:00 +0000 Subject: [PATCH 22/28] appended line to file --- composer/cicd_sample/dags/example2_dag.py | 1 + composer/cicd_sample/utils/deploy_dags_from_diff_test.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/composer/cicd_sample/dags/example2_dag.py b/composer/cicd_sample/dags/example2_dag.py index 9dc9620202e..7d3f4c8ce84 100644 --- a/composer/cicd_sample/dags/example2_dag.py +++ b/composer/cicd_sample/dags/example2_dag.py @@ -49,3 +49,4 @@ # appended line # appended line # appended line +# appended line diff --git a/composer/cicd_sample/utils/deploy_dags_from_diff_test.py b/composer/cicd_sample/utils/deploy_dags_from_diff_test.py index de808c67607..2ce2c13f7f3 100644 --- a/composer/cicd_sample/utils/deploy_dags_from_diff_test.py +++ b/composer/cicd_sample/utils/deploy_dags_from_diff_test.py @@ -36,7 +36,7 @@ def test_create_dags_list_with_changes() -> None: this test checks for :return: """ - full_dag_dir_str = str(DAGS_DIR) # .resolve()) + full_dag_dir_str = str(DAGS_DIR.name) # .resolve()) full_repo_root_str = str(REPO_ROOT.resolve()) print(full_dag_dir_str, full_repo_root_str) repo = git.Repo(REPO_ROOT) From 827ef85d868aff190758de6e3d1f643f9c709c37 Mon Sep 17 00:00:00 2001 From: Alexander Shires Date: Thu, 16 Feb 2023 11:11:47 +0000 Subject: [PATCH 23/28] appended line to file --- composer/cicd_sample/dags/example2_dag.py | 1 + composer/cicd_sample/utils/deploy_dags_from_diff_test.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/composer/cicd_sample/dags/example2_dag.py b/composer/cicd_sample/dags/example2_dag.py index 7d3f4c8ce84..b80b3061c50 100644 --- a/composer/cicd_sample/dags/example2_dag.py +++ b/composer/cicd_sample/dags/example2_dag.py @@ -50,3 +50,4 @@ # appended line # appended line # appended line +# appended line diff --git a/composer/cicd_sample/utils/deploy_dags_from_diff_test.py b/composer/cicd_sample/utils/deploy_dags_from_diff_test.py index 2ce2c13f7f3..521fc01a4df 100644 --- a/composer/cicd_sample/utils/deploy_dags_from_diff_test.py +++ b/composer/cicd_sample/utils/deploy_dags_from_diff_test.py @@ -36,7 +36,7 @@ def test_create_dags_list_with_changes() -> None: this test checks for :return: """ - full_dag_dir_str = str(DAGS_DIR.name) # .resolve()) + full_dag_dir_str = "cicd/dags" # str(DAGS_DIR) # .resolve()) full_repo_root_str = str(REPO_ROOT.resolve()) print(full_dag_dir_str, full_repo_root_str) repo = git.Repo(REPO_ROOT) From 286bc5c0791e342ad7254a89c23a8fc68e709c28 Mon Sep 17 00:00:00 2001 From: Alexander Shires Date: Thu, 16 Feb 2023 11:12:04 +0000 Subject: [PATCH 24/28] appended line to file --- composer/cicd_sample/dags/example2_dag.py | 1 + composer/cicd_sample/utils/deploy_dags_from_diff_test.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/composer/cicd_sample/dags/example2_dag.py b/composer/cicd_sample/dags/example2_dag.py index b80b3061c50..4dc67d725bc 100644 --- a/composer/cicd_sample/dags/example2_dag.py +++ b/composer/cicd_sample/dags/example2_dag.py @@ -51,3 +51,4 @@ # appended line # appended line # appended line +# appended line diff --git a/composer/cicd_sample/utils/deploy_dags_from_diff_test.py b/composer/cicd_sample/utils/deploy_dags_from_diff_test.py index 521fc01a4df..4618afc8697 100644 --- a/composer/cicd_sample/utils/deploy_dags_from_diff_test.py +++ b/composer/cicd_sample/utils/deploy_dags_from_diff_test.py @@ -36,7 +36,7 @@ def test_create_dags_list_with_changes() -> None: this test checks for :return: """ - full_dag_dir_str = "cicd/dags" # str(DAGS_DIR) # .resolve()) + full_dag_dir_str = "cicd_sample/dags" # str(DAGS_DIR) # .resolve()) full_repo_root_str = str(REPO_ROOT.resolve()) print(full_dag_dir_str, full_repo_root_str) repo = git.Repo(REPO_ROOT) From 18cee69b8c44e8ed8432acd2939c20e88f9a7862 Mon Sep 17 00:00:00 2001 From: Alexander Shires Date: Thu, 16 Feb 2023 11:13:39 +0000 Subject: [PATCH 25/28] appended line to file --- composer/cicd_sample/dags/example2_dag.py | 1 + composer/cicd_sample/utils/deploy_dags_from_diff_test.py | 1 + 2 files changed, 2 insertions(+) diff --git a/composer/cicd_sample/dags/example2_dag.py b/composer/cicd_sample/dags/example2_dag.py index 4dc67d725bc..6fdf691e38f 100644 --- a/composer/cicd_sample/dags/example2_dag.py +++ b/composer/cicd_sample/dags/example2_dag.py @@ -52,3 +52,4 @@ # appended line # appended line # appended line +# appended line diff --git a/composer/cicd_sample/utils/deploy_dags_from_diff_test.py b/composer/cicd_sample/utils/deploy_dags_from_diff_test.py index 4618afc8697..45f8bc1a73b 100644 --- a/composer/cicd_sample/utils/deploy_dags_from_diff_test.py +++ b/composer/cicd_sample/utils/deploy_dags_from_diff_test.py @@ -25,6 +25,7 @@ import deploy_dags_from_diff # noqa: I100 - lint is incorrectly saying this is out of order +# DAGS_DIR = "cicd_sample/dags" # this needs to be relative to the REPO roo DAGS_DIR = pathlib.Path(__file__).parent.parent / "dags/" REPO_ROOT = pathlib.Path(__file__).parent.parent.parent.parent REPO_MAIN = "main" From 6d4aa21bf7e592254ddfaee14409aabf2f0c38c1 Mon Sep 17 00:00:00 2001 From: Alexander Shires Date: Thu, 16 Feb 2023 11:13:57 +0000 Subject: [PATCH 26/28] appended line to file --- composer/cicd_sample/dags/example2_dag.py | 1 + composer/cicd_sample/utils/deploy_dags_from_diff_test.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/composer/cicd_sample/dags/example2_dag.py b/composer/cicd_sample/dags/example2_dag.py index 6fdf691e38f..d596abdb4c7 100644 --- a/composer/cicd_sample/dags/example2_dag.py +++ b/composer/cicd_sample/dags/example2_dag.py @@ -53,3 +53,4 @@ # appended line # appended line # appended line +# appended line diff --git a/composer/cicd_sample/utils/deploy_dags_from_diff_test.py b/composer/cicd_sample/utils/deploy_dags_from_diff_test.py index 45f8bc1a73b..7bfc3507740 100644 --- a/composer/cicd_sample/utils/deploy_dags_from_diff_test.py +++ b/composer/cicd_sample/utils/deploy_dags_from_diff_test.py @@ -52,7 +52,7 @@ def test_create_dags_list_with_changes() -> None: full_dag_dir_str, full_repo_root_str, REPO_MAIN) print(dag_list) assert len(dag_list) > 0 - assert "example2_dag.py" in dag_list[0] + assert "example2_dag.py" in dag_list[0].name # cleanup - uncomment once test dev is done # repo.git.checkout(REPO_MAIN) From ab21fc787bb742576be02c4c54fea9f836163749 Mon Sep 17 00:00:00 2001 From: Alexander Shires Date: Thu, 16 Feb 2023 11:14:21 +0000 Subject: [PATCH 27/28] appended line to file --- composer/cicd_sample/dags/example2_dag.py | 1 + composer/cicd_sample/utils/deploy_dags_from_diff_test.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/composer/cicd_sample/dags/example2_dag.py b/composer/cicd_sample/dags/example2_dag.py index d596abdb4c7..1f5b1301d97 100644 --- a/composer/cicd_sample/dags/example2_dag.py +++ b/composer/cicd_sample/dags/example2_dag.py @@ -54,3 +54,4 @@ # appended line # appended line # appended line +# appended line diff --git a/composer/cicd_sample/utils/deploy_dags_from_diff_test.py b/composer/cicd_sample/utils/deploy_dags_from_diff_test.py index 7bfc3507740..1bd6b81006d 100644 --- a/composer/cicd_sample/utils/deploy_dags_from_diff_test.py +++ b/composer/cicd_sample/utils/deploy_dags_from_diff_test.py @@ -62,6 +62,9 @@ def test_create_dags_list_no_changes() -> None: this test checks for an empty list based on no changes in git :return: """ - dag_list = deploy_dags_from_diff.create_dags_list_from_git_diff(DAGS_DIR, REPO_ROOT, REPO_MAIN) + full_dag_dir_str = "cicd_sample/dags" # str(DAGS_DIR) # .resolve()) + full_repo_root_str = str(REPO_ROOT.resolve()) + dag_list = deploy_dags_from_diff.create_dags_list_from_git_diff( + full_dag_dir_str, full_repo_root_str, REPO_MAIN) assert len(dag_list) == 0 From df434a13391c0842381500e2a6bd6b41585f5ccb Mon Sep 17 00:00:00 2001 From: Alexander Shires Date: Thu, 16 Feb 2023 11:18:41 +0000 Subject: [PATCH 28/28] remove test appends --- composer/cicd_sample/dags/example2_dag.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/composer/cicd_sample/dags/example2_dag.py b/composer/cicd_sample/dags/example2_dag.py index 1f5b1301d97..81996663d8a 100644 --- a/composer/cicd_sample/dags/example2_dag.py +++ b/composer/cicd_sample/dags/example2_dag.py @@ -44,14 +44,4 @@ print_dag_run_conf = bash.BashOperator( task_id="print_dag_run_conf", bash_command="echo {{ dag_run.id }}" ) -# appended line# appended line# appended line# appended line# appended line# appended line# appended line -# appended line -# appended line -# appended line -# appended line -# appended line -# appended line -# appended line -# appended line -# appended line -# appended line +