diff --git a/docs/technical_documentation/overview.md b/docs/technical_documentation/overview.md index 880040d..b239674 100644 --- a/docs/technical_documentation/overview.md +++ b/docs/technical_documentation/overview.md @@ -37,8 +37,8 @@ This component is an imported library which is shared across multiple GitHub too ### Historic Usage Data -This section gathers data from AWS S3. The Copilot usage endpoints have a limitation where they only return the last 100 days worth of information. To get around this, the project has an AWS Lambda function which runs weekly and stores data within an S3 bucket. +This section gathers data from AWS S3. The Copilot usage endpoints have a limitation where they only return the last 28 days worth of information. To get around this, the project has an AWS Lambda function which runs weekly and stores data within an S3 bucket. -### Copilot Teams Data +### Copilot Teams Data (Deprecated - functionality removed but may be restored via alternative methods) This section gathers a list of teams within the organisation with Copilot data and updates the S3 bucket accordingly. This allows all relevant teams to be displayed within the dashboard. diff --git a/docs/technical_documentation/team_usage.md b/docs/technical_documentation/team_usage.md index cc43d8f..b50bf39 100644 --- a/docs/technical_documentation/team_usage.md +++ b/docs/technical_documentation/team_usage.md @@ -1,5 +1,7 @@ # Copilot Team Usage +Note: This functionality has been removed as of 19th March 2026 as the endpoint used to fetch metrics for team usage is being sunsetted. However, it may be restored via alternative methods in the future. + ## Overview This section of the project leverages GitHub OAuth2 for user authentication, granting access to essential data. diff --git a/poetry.lock b/poetry.lock index 5a33af3..78d2ae2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -49,48 +49,54 @@ extras = ["regex"] [[package]] name = "black" -version = "24.10.0" +version = "26.3.1" description = "The uncompromising code formatter." optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["dev"] files = [ - {file = "black-24.10.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e6668650ea4b685440857138e5fe40cde4d652633b1bdffc62933d0db4ed9812"}, - {file = "black-24.10.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1c536fcf674217e87b8cc3657b81809d3c085d7bf3ef262ead700da345bfa6ea"}, - {file = "black-24.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:649fff99a20bd06c6f727d2a27f401331dc0cc861fb69cde910fe95b01b5928f"}, - {file = "black-24.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:fe4d6476887de70546212c99ac9bd803d90b42fc4767f058a0baa895013fbb3e"}, - {file = "black-24.10.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5a2221696a8224e335c28816a9d331a6c2ae15a2ee34ec857dcf3e45dbfa99ad"}, - {file = "black-24.10.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f9da3333530dbcecc1be13e69c250ed8dfa67f43c4005fb537bb426e19200d50"}, - {file = "black-24.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4007b1393d902b48b36958a216c20c4482f601569d19ed1df294a496eb366392"}, - {file = "black-24.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:394d4ddc64782e51153eadcaaca95144ac4c35e27ef9b0a42e121ae7e57a9175"}, - {file = "black-24.10.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b5e39e0fae001df40f95bd8cc36b9165c5e2ea88900167bddf258bacef9bbdc3"}, - {file = "black-24.10.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d37d422772111794b26757c5b55a3eade028aa3fde43121ab7b673d050949d65"}, - {file = "black-24.10.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:14b3502784f09ce2443830e3133dacf2c0110d45191ed470ecb04d0f5f6fcb0f"}, - {file = "black-24.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:30d2c30dc5139211dda799758559d1b049f7f14c580c409d6ad925b74a4208a8"}, - {file = "black-24.10.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1cbacacb19e922a1d75ef2b6ccaefcd6e93a2c05ede32f06a21386a04cedb981"}, - {file = "black-24.10.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1f93102e0c5bb3907451063e08b9876dbeac810e7da5a8bfb7aeb5a9ef89066b"}, - {file = "black-24.10.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ddacb691cdcdf77b96f549cf9591701d8db36b2f19519373d60d31746068dbf2"}, - {file = "black-24.10.0-cp313-cp313-win_amd64.whl", hash = "sha256:680359d932801c76d2e9c9068d05c6b107f2584b2a5b88831c83962eb9984c1b"}, - {file = "black-24.10.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:17374989640fbca88b6a448129cd1745c5eb8d9547b464f281b251dd00155ccd"}, - {file = "black-24.10.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:63f626344343083322233f175aaf372d326de8436f5928c042639a4afbbf1d3f"}, - {file = "black-24.10.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfa1d0cb6200857f1923b602f978386a3a2758a65b52e0950299ea014be6800"}, - {file = "black-24.10.0-cp39-cp39-win_amd64.whl", hash = "sha256:2cd9c95431d94adc56600710f8813ee27eea544dd118d45896bb734e9d7a0dc7"}, - {file = "black-24.10.0-py3-none-any.whl", hash = "sha256:3bb2b7a1f7b685f85b11fed1ef10f8a9148bceb49853e47a294a3dd963c1dd7d"}, - {file = "black-24.10.0.tar.gz", hash = "sha256:846ea64c97afe3bc677b761787993be4991810ecc7a4a937816dd6bddedc4875"}, + {file = "black-26.3.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:86a8b5035fce64f5dcd1b794cf8ec4d31fe458cf6ce3986a30deb434df82a1d2"}, + {file = "black-26.3.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5602bdb96d52d2d0672f24f6ffe5218795736dd34807fd0fd55ccd6bf206168b"}, + {file = "black-26.3.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6c54a4a82e291a1fee5137371ab488866b7c86a3305af4026bdd4dc78642e1ac"}, + {file = "black-26.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:6e131579c243c98f35bce64a7e08e87fb2d610544754675d4a0e73a070a5aa3a"}, + {file = "black-26.3.1-cp310-cp310-win_arm64.whl", hash = "sha256:5ed0ca58586c8d9a487352a96b15272b7fa55d139fc8496b519e78023a8dab0a"}, + {file = "black-26.3.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:28ef38aee69e4b12fda8dba75e21f9b4f979b490c8ac0baa7cb505369ac9e1ff"}, + {file = "black-26.3.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bf9bf162ed91a26f1adba8efda0b573bc6924ec1408a52cc6f82cb73ec2b142c"}, + {file = "black-26.3.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:474c27574d6d7037c1bc875a81d9be0a9a4f9ee95e62800dab3cfaadbf75acd5"}, + {file = "black-26.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:5e9d0d86df21f2e1677cc4bd090cd0e446278bcbbe49bf3659c308c3e402843e"}, + {file = "black-26.3.1-cp311-cp311-win_arm64.whl", hash = "sha256:9a5e9f45e5d5e1c5b5c29b3bd4265dcc90e8b92cf4534520896ed77f791f4da5"}, + {file = "black-26.3.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b5e6f89631eb88a7302d416594a32faeee9fb8fb848290da9d0a5f2903519fc1"}, + {file = "black-26.3.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:41cd2012d35b47d589cb8a16faf8a32ef7a336f56356babd9fcf70939ad1897f"}, + {file = "black-26.3.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f76ff19ec5297dd8e66eb64deda23631e642c9393ab592826fd4bdc97a4bce7"}, + {file = "black-26.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:ddb113db38838eb9f043623ba274cfaf7d51d5b0c22ecb30afe58b1bb8322983"}, + {file = "black-26.3.1-cp312-cp312-win_arm64.whl", hash = "sha256:dfdd51fc3e64ea4f35873d1b3fb25326773d55d2329ff8449139ebaad7357efb"}, + {file = "black-26.3.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:855822d90f884905362f602880ed8b5df1b7e3ee7d0db2502d4388a954cc8c54"}, + {file = "black-26.3.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8a33d657f3276328ce00e4d37fe70361e1ec7614da5d7b6e78de5426cb56332f"}, + {file = "black-26.3.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f1cd08e99d2f9317292a311dfe578fd2a24b15dbce97792f9c4d752275c1fa56"}, + {file = "black-26.3.1-cp313-cp313-win_amd64.whl", hash = "sha256:c7e72339f841b5a237ff14f7d3880ddd0fc7f98a1199e8c4327f9a4f478c1839"}, + {file = "black-26.3.1-cp313-cp313-win_arm64.whl", hash = "sha256:afc622538b430aa4c8c853f7f63bc582b3b8030fd8c80b70fb5fa5b834e575c2"}, + {file = "black-26.3.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2d6bfaf7fd0993b420bed691f20f9492d53ce9a2bcccea4b797d34e947318a78"}, + {file = "black-26.3.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:f89f2ab047c76a9c03f78d0d66ca519e389519902fa27e7a91117ef7611c0568"}, + {file = "black-26.3.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b07fc0dab849d24a80a29cfab8d8a19187d1c4685d8a5e6385a5ce323c1f015f"}, + {file = "black-26.3.1-cp314-cp314-win_amd64.whl", hash = "sha256:0126ae5b7c09957da2bdbd91a9ba1207453feada9e9fe51992848658c6c8e01c"}, + {file = "black-26.3.1-cp314-cp314-win_arm64.whl", hash = "sha256:92c0ec1f2cc149551a2b7b47efc32c866406b6891b0ee4625e95967c8f4acfb1"}, + {file = "black-26.3.1-py3-none-any.whl", hash = "sha256:2bd5aa94fc267d38bb21a70d7410a89f1a1d318841855f698746f8e7f51acd1b"}, + {file = "black-26.3.1.tar.gz", hash = "sha256:2c50f5063a9641c7eed7795014ba37b0f5fa227f3d408b968936e24bc0566b07"}, ] [package.dependencies] click = ">=8.0.0" mypy-extensions = ">=0.4.3" packaging = ">=22.0" -pathspec = ">=0.9.0" +pathspec = ">=1.0.0" platformdirs = ">=2" +pytokens = ">=0.4.0,<0.5.0" [package.extras] colorama = ["colorama (>=0.4.3)"] d = ["aiohttp (>=3.10)"] jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] -uvloop = ["uvloop (>=0.15.2)"] +uvloop = ["uvloop (>=0.15.2) ; sys_platform != \"win32\"", "winloop (>=0.5.0) ; sys_platform == \"win32\""] [[package]] name = "boto3" @@ -369,7 +375,7 @@ files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] -markers = {dev = "platform_system == \"Windows\" or sys_platform == \"win32\""} +markers = {dev = "sys_platform == \"win32\" or platform_system == \"Windows\""} [[package]] name = "coverage" @@ -1082,7 +1088,7 @@ version = "1.0.4" description = "Utility library for gitignore style pattern matching of file paths." optional = false python-versions = ">=3.9" -groups = ["main", "dev", "docs"] +groups = ["dev", "docs"] files = [ {file = "pathspec-1.0.4-py3-none-any.whl", hash = "sha256:fb6ae2fd4e7c921a165808a552060e722767cfa526f99ca5156ed2ce45a5c723"}, {file = "pathspec-1.0.4.tar.gz", hash = "sha256:0210e2ae8a21a9137c0d470578cb0e595af87edaa6ebf12ff176f14a02e0e645"}, @@ -1276,13 +1282,68 @@ files = [ [package.dependencies] six = ">=1.5" +[[package]] +name = "pytokens" +version = "0.4.1" +description = "A Fast, spec compliant Python 3.14+ tokenizer that runs on older Pythons." +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "pytokens-0.4.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2a44ed93ea23415c54f3face3b65ef2b844d96aeb3455b8a69b3df6beab6acc5"}, + {file = "pytokens-0.4.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:add8bf86b71a5d9fb5b89f023a80b791e04fba57960aa790cc6125f7f1d39dfe"}, + {file = "pytokens-0.4.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:670d286910b531c7b7e3c0b453fd8156f250adb140146d234a82219459b9640c"}, + {file = "pytokens-0.4.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:4e691d7f5186bd2842c14813f79f8884bb03f5995f0575272009982c5ac6c0f7"}, + {file = "pytokens-0.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:27b83ad28825978742beef057bfe406ad6ed524b2d28c252c5de7b4a6dd48fa2"}, + {file = "pytokens-0.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d70e77c55ae8380c91c0c18dea05951482e263982911fc7410b1ffd1dadd3440"}, + {file = "pytokens-0.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a58d057208cb9075c144950d789511220b07636dd2e4708d5645d24de666bdc"}, + {file = "pytokens-0.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b49750419d300e2b5a3813cf229d4e5a4c728dae470bcc89867a9ad6f25a722d"}, + {file = "pytokens-0.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d9907d61f15bf7261d7e775bd5d7ee4d2930e04424bab1972591918497623a16"}, + {file = "pytokens-0.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:ee44d0f85b803321710f9239f335aafe16553b39106384cef8e6de40cb4ef2f6"}, + {file = "pytokens-0.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:140709331e846b728475786df8aeb27d24f48cbcf7bcd449f8de75cae7a45083"}, + {file = "pytokens-0.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d6c4268598f762bc8e91f5dbf2ab2f61f7b95bdc07953b602db879b3c8c18e1"}, + {file = "pytokens-0.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:24afde1f53d95348b5a0eb19488661147285ca4dd7ed752bbc3e1c6242a304d1"}, + {file = "pytokens-0.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ad948d085ed6c16413eb5fec6b3e02fa00dc29a2534f088d3302c47eb59adf9"}, + {file = "pytokens-0.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:3f901fe783e06e48e8cbdc82d631fca8f118333798193e026a50ce1b3757ea68"}, + {file = "pytokens-0.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8bdb9d0ce90cbf99c525e75a2fa415144fd570a1ba987380190e8b786bc6ef9b"}, + {file = "pytokens-0.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5502408cab1cb18e128570f8d598981c68a50d0cbd7c61312a90507cd3a1276f"}, + {file = "pytokens-0.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:29d1d8fb1030af4d231789959f21821ab6325e463f0503a61d204343c9b355d1"}, + {file = "pytokens-0.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:970b08dd6b86058b6dc07efe9e98414f5102974716232d10f32ff39701e841c4"}, + {file = "pytokens-0.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:9bd7d7f544d362576be74f9d5901a22f317efc20046efe2034dced238cbbfe78"}, + {file = "pytokens-0.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4a14d5f5fc78ce85e426aa159489e2d5961acf0e47575e08f35584009178e321"}, + {file = "pytokens-0.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97f50fd18543be72da51dd505e2ed20d2228c74e0464e4262e4899797803d7fa"}, + {file = "pytokens-0.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dc74c035f9bfca0255c1af77ddd2d6ae8419012805453e4b0e7513e17904545d"}, + {file = "pytokens-0.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f66a6bbe741bd431f6d741e617e0f39ec7257ca1f89089593479347cc4d13324"}, + {file = "pytokens-0.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:b35d7e5ad269804f6697727702da3c517bb8a5228afa450ab0fa787732055fc9"}, + {file = "pytokens-0.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8fcb9ba3709ff77e77f1c7022ff11d13553f3c30299a9fe246a166903e9091eb"}, + {file = "pytokens-0.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:79fc6b8699564e1f9b521582c35435f1bd32dd06822322ec44afdeba666d8cb3"}, + {file = "pytokens-0.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d31b97b3de0f61571a124a00ffe9a81fb9939146c122c11060725bd5aea79975"}, + {file = "pytokens-0.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:967cf6e3fd4adf7de8fc73cd3043754ae79c36475c1c11d514fc72cf5490094a"}, + {file = "pytokens-0.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:584c80c24b078eec1e227079d56dc22ff755e0ba8654d8383b2c549107528918"}, + {file = "pytokens-0.4.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:da5baeaf7116dced9c6bb76dc31ba04a2dc3695f3d9f74741d7910122b456edc"}, + {file = "pytokens-0.4.1-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:11edda0942da80ff58c4408407616a310adecae1ddd22eef8c692fe266fa5009"}, + {file = "pytokens-0.4.1-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0fc71786e629cef478cbf29d7ea1923299181d0699dbe7c3c0f4a583811d9fc1"}, + {file = "pytokens-0.4.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:dcafc12c30dbaf1e2af0490978352e0c4041a7cde31f4f81435c2a5e8b9cabb6"}, + {file = "pytokens-0.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:42f144f3aafa5d92bad964d471a581651e28b24434d184871bd02e3a0d956037"}, + {file = "pytokens-0.4.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:34bcc734bd2f2d5fe3b34e7b3c0116bfb2397f2d9666139988e7a3eb5f7400e3"}, + {file = "pytokens-0.4.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:941d4343bf27b605e9213b26bfa1c4bf197c9c599a9627eb7305b0defcfe40c1"}, + {file = "pytokens-0.4.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3ad72b851e781478366288743198101e5eb34a414f1d5627cdd585ca3b25f1db"}, + {file = "pytokens-0.4.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:682fa37ff4d8e95f7df6fe6fe6a431e8ed8e788023c6bcc0f0880a12eab80ad1"}, + {file = "pytokens-0.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:30f51edd9bb7f85c748979384165601d028b84f7bd13fe14d3e065304093916a"}, + {file = "pytokens-0.4.1-py3-none-any.whl", hash = "sha256:26cef14744a8385f35d0e095dc8b3a7583f6c953c2e3d269c7f82484bf5ad2de"}, + {file = "pytokens-0.4.1.tar.gz", hash = "sha256:292052fe80923aae2260c073f822ceba21f3872ced9a68bb7953b348e561179a"}, +] + +[package.extras] +dev = ["black", "build", "mypy", "pytest", "pytest-cov", "setuptools", "tox", "twine", "wheel"] + [[package]] name = "pyyaml" version = "6.0.2" description = "YAML parser and emitter for Python" optional = false python-versions = ">=3.8" -groups = ["main", "docs"] +groups = ["docs"] files = [ {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"}, {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"}, @@ -1519,26 +1580,7 @@ files = [ [package.extras] watchmedo = ["PyYAML (>=3.10)"] -[[package]] -name = "yamllint" -version = "1.38.0" -description = "A linter for YAML files." -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "yamllint-1.38.0-py3-none-any.whl", hash = "sha256:fc394a5b3be980a4062607b8fdddc0843f4fa394152b6da21722f5d59013c220"}, - {file = "yamllint-1.38.0.tar.gz", hash = "sha256:09e5f29531daab93366bb061e76019d5e91691ef0a40328f04c927387d1d364d"}, -] - -[package.dependencies] -pathspec = ">=1.0.0" -pyyaml = "*" - -[package.extras] -dev = ["doc8", "flake8", "flake8-import-order", "rstcheck[sphinx]", "ruff", "sphinx"] - [metadata] lock-version = "2.1" python-versions = "^3.12" -content-hash = "8cb6c4a88a27848571078e5ab89f980e1daf60ac2a35842524d155dd3d21c819" +content-hash = "bbeda2848b9b4b6ee321fe0d887bd8e90c30f386ff1bcf5de394200cb3affcc4" diff --git a/pyproject.toml b/pyproject.toml index b336067..e734db3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,13 +25,13 @@ six = "^1.17.0" urllib3 = "^2.6.3" [tool.poetry.group.dev.dependencies] -black = "^24.8.0" ruff = "^0.6.5" pylint = "^3.2.7" mypy = "^1.11.2" pytest = "^8.4.1" pytest-cov = "^6.2.1" pytest-xdist = "^3.8.0" +black = "^26.3.1" [tool.poetry.group.docs.dependencies] mkdocs = "^1.6.0" diff --git a/src/main.py b/src/main.py index d75891a..92daa3f 100644 --- a/src/main.py +++ b/src/main.py @@ -8,12 +8,12 @@ import json import logging import os -from typing import Any, Optional +from typing import Any import boto3 import github_api_toolkit +import requests from botocore.exceptions import ClientError -from requests import Response # GitHub Organisation org = os.getenv("GITHUB_ORG") @@ -29,7 +29,7 @@ # AWS Bucket Path BUCKET_NAME = f"{account}-copilot-usage-dashboard" -OBJECT_NAME = "historic_usage_data.json" +OBJECT_NAME = "org_history.json" logger = logging.getLogger() @@ -57,64 +57,6 @@ # } -def get_copilot_team_date(gh: github_api_toolkit.github_interface, page: int) -> list: - """Gets a list of GitHub Teams with Copilot Data for a given API page. - - Args: - gh (github_api_toolkit.github_interface): An instance of the github_interface class. - page (int): The page number of the API request. - - Returns: - list: A list of GitHub Teams with Copilot Data. - """ - copilot_teams = [] - - response = gh.get(f"/orgs/{org}/teams", params={"per_page": 100, "page": page}) - teams = response.json() - for team in teams: - usage_data = gh.get(f"/orgs/{org}/team/{team['name']}/copilot/metrics") - - if not isinstance(usage_data, Response): - - # If the response is not a Response object, no copilot data is available for this team - # We can then skip this team - - # We don't log this as an error, as it is expected and it'd be too noisy within logs - - continue - - # If the response has data, append the team to the list - # If there is no data, .json() will return an empty list - if usage_data.json(): - - team_name = team.get("name", "") - team_slug = team.get("slug", "") - team_description = team.get("description", "") - team_html_url = team.get("html_url", "") - - logger.info( - "Team %s has Copilot data", - team_name, - extra={ - "team_name": team_name, - "team_slug": team_slug, - "team_description": team_description, - "team_html_url": team_html_url, - }, - ) - - copilot_teams.append( - { - "name": team_name, - "slug": team_slug, - "description": team_description, - "url": team_html_url, - } - ) - - return copilot_teams - - def get_and_update_historic_usage( s3: boto3.client, gh: github_api_toolkit.github_interface, write_data_locally: bool ) -> tuple: @@ -129,8 +71,8 @@ def get_and_update_historic_usage( tuple: A tuple containing the updated historic usage data and a list of dates added. """ # Get the usage data - usage_data = gh.get(f"/orgs/{org}/copilot/metrics") - usage_data = usage_data.json() + api_response = gh.get(f"/orgs/{org}/copilot/metrics/reports/organization-28-day/latest").json() + usage_data = requests.get(api_response["download_links"][0], timeout=30).json()["day_totals"] logger.info("Usage data retrieved") @@ -139,133 +81,36 @@ def get_and_update_historic_usage( historic_usage = json.loads(response["Body"].read().decode("utf-8")) except ClientError as e: logger.error("Error getting %s: %s. Using empty list.", OBJECT_NAME, e) - historic_usage = [] dates_added = [] - # Append the new usage data to the historic_usage_data.json - for date in usage_data: - if not any(d["date"] == date["date"] for d in historic_usage): - historic_usage.append(date) - - dates_added.append(date["date"]) + for day in usage_data: + if not any(d["day"] == day["day"] for d in historic_usage): + historic_usage.append(day) + dates_added.append(day["day"]) + logger.info("Added data for day %s", day["day"]) - logger.info( - "New usage data added to %s", - OBJECT_NAME, - extra={"no_days_added": len(dates_added), "dates_added": dates_added}, - ) + sorted_historic_usage = sorted(historic_usage, key=lambda x: x["day"]) if not write_data_locally: # Write the updated historic_usage to historic_usage_data.json - update_s3_object(s3, BUCKET_NAME, OBJECT_NAME, historic_usage) + update_s3_object(s3, BUCKET_NAME, OBJECT_NAME, sorted_historic_usage) else: local_path = f"output/{OBJECT_NAME}" os.makedirs("output", exist_ok=True) with open(local_path, "w", encoding="utf-8") as f: - json.dump(historic_usage, f, indent=4) + json.dump(sorted_historic_usage, f, indent=4) logger.info("Historic usage data written locally to %s (S3 skipped)", local_path) - return historic_usage, dates_added - - -def get_and_update_copilot_teams( - s3: boto3.client, gh: github_api_toolkit.github_interface, write_data_locally: bool -) -> list: - """Get and update GitHub Teams with Copilot Data. - - Args: - s3 (boto3.client): An S3 client. - gh (github_api_toolkit.github_interface): An instance of the github_interface class. - write_data_locally (bool): Whether to write data locally instead of to an S3 bucket. - - Returns: - list: A list of GitHub Teams with Copilot Data. - """ - logger.info("Getting GitHub Teams with Copilot Data") - - copilot_teams = [] - - response = gh.get(f"/orgs/{org}/teams", params={"per_page": 100}) - - # Get the last page of teams - try: - last_page = int(response.links["last"]["url"].split("=")[-1]) - except KeyError: - last_page = 1 - - for page in range(1, last_page + 1): - page_teams = get_copilot_team_date(gh, page) - - copilot_teams = copilot_teams + page_teams - logger.info( - "Fetched GitHub Teams with Copilot Data", - extra={"no_teams": len(copilot_teams)}, + "Usage data written to %s: %d days added (%s)", + OBJECT_NAME, + len(dates_added), + dates_added, ) - if not write_data_locally: - update_s3_object(s3, BUCKET_NAME, "copilot_teams.json", copilot_teams) - else: - local_path = "output/copilot_teams.json" - os.makedirs("output", exist_ok=True) - with open(local_path, "w", encoding="utf-8") as f: - json.dump(copilot_teams, f, indent=4) - logger.info("Copilot teams data written locally to %s (S3 skipped)", local_path) - - return copilot_teams - - -def create_dictionary( - gh: github_api_toolkit.github_interface, copilot_teams: list, existing_team_history: list -) -> list: - """Create a dictionary for quick lookup of existing team data using the `name` field. - - Args: - gh (github_api_toolkit.github_interface): An instance of the github_interface class. - copilot_teams (list): List of teams with Copilot data. - existing_team_history (list): List of existing team history data. - - Returns: - list: A list of dictionaries containing team data and their history. - """ - existing_team_data_map = { - single_team["team"]["name"]: single_team for single_team in existing_team_history - } - - # Iterate through identified teams - for team in copilot_teams: - team_name = team.get("name", "") - if not team_name: - logger.warning("Skipping team with no name") - continue - - # Determine the last known date for the team - last_known_date = None - if team_name in existing_team_data_map: - existing_dates = [entry["date"] for entry in existing_team_data_map[team_name]["data"]] - if existing_dates: - last_known_date = max(existing_dates) # Get the most recent date - - # Assign the last known date to the `since` query parameter - query_params = {} - if last_known_date: - query_params["since"] = last_known_date - - single_team_history = get_team_history(gh, team_name, query_params) - if not single_team_history: - logger.info("No new history found for team %s", team_name) - continue - - # Append new data to the existing team history - new_team_data = single_team_history - if team_name in existing_team_data_map: - existing_team_data_map[team_name]["data"].extend(new_team_data) - else: - existing_team_data_map[team_name] = {"team": team, "data": new_team_data} - - return list(existing_team_data_map.values()) + return sorted_historic_usage, dates_added def update_s3_object( @@ -298,31 +143,6 @@ def update_s3_object( return False -def get_team_history( - gh: github_api_toolkit.github_interface, team: str, query_params: Optional[dict] = None -) -> list[dict]: - """Gets the team metrics Copilot data through the API. - Note - This endpoint will only return results for a given day if the team had - five or more members with active Copilot licenses on that day, - as evaluated at the end of that day. - - Args: - gh (github_api_toolkit.github_interface): An instance of the github_interface class. - team (str): Team name. - query_params (dict): Additional query parameters for the API request. - - Returns: - list[dict]: A team's GitHub Copilot metrics or None if an error occurs. - """ - response = gh.get(f"/orgs/{org}/team/{team}/copilot/metrics", params=query_params) - - if not isinstance(response, Response): - # If the response is not a Response object, no copilot data is available for this team - # We can return None which is then handled by the calling function - return None - return response.json() - - def get_dict_value(dictionary: dict, key: str) -> Any: """Gets a value from a dictionary and raises an exception if it is not found. @@ -406,6 +226,13 @@ def handler(event: dict, context) -> str: # pylint: disable=unused-argument, to logging.basicConfig( filename="debug.log", filemode="w", + format="%(asctime)s %(levelname)s %(message)s", + ) + else: + # Ensure INFO logs show in the terminal when not logging to a file + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(message)s", ) # Create an S3 client @@ -437,37 +264,6 @@ def handler(event: dict, context) -> str: # pylint: disable=unused-argument, to # Copilot Usage Data (Historic) historic_usage, dates_added = get_and_update_historic_usage(s3, gh, write_data_locally) - # GitHub Teams with Copilot Data - copilot_teams = get_and_update_copilot_teams(s3, gh, write_data_locally) - - logger.info("Getting history of each team identified previously") - - # Retrieve existing team history from S3 - try: - response = s3.get_object(Bucket=BUCKET_NAME, Key="teams_history.json") - existing_team_history = json.loads(response["Body"].read().decode("utf-8")) - except ClientError as e: - logger.warning("Error retrieving existing team history: %s", e) - existing_team_history = [] - - logger.info("Existing team history has %d entries", len(existing_team_history)) - - if not write_data_locally: - # Convert to dictionary for quick lookup - updated_team_history = create_dictionary(gh, copilot_teams, existing_team_history) - - # Write updated team history to S3 - # This line isn't covered by tests as it's painful to get to. - # The function itself is tested though. - update_s3_object(s3, BUCKET_NAME, "teams_history.json", updated_team_history) - else: - local_path = "output/teams_history.json" - os.makedirs("output", exist_ok=True) - updated_team_history = create_dictionary(gh, copilot_teams, existing_team_history) - with open(local_path, "w", encoding="utf-8") as f: - json.dump(updated_team_history, f, indent=4) - logger.info("Team history written locally to %s (S3 skipped)", local_path) - logger.info( "Process complete", extra={ @@ -476,14 +272,13 @@ def handler(event: dict, context) -> str: # pylint: disable=unused-argument, to "dates_added": dates_added, "no_dates_before": len(historic_usage) - len(dates_added), "no_dates_after": len(historic_usage), - "no_copilot_teams": len(copilot_teams), }, ) return "Github Data logging is now complete." -# # Dev Only -# # Uncomment the following line to run the script locally +# Dev Only +# Uncomment the following line to run the script locally # if __name__ == "__main__": # handler(None, None) diff --git a/tests/test_main.py b/tests/test_main.py index 628f57f..5b70772 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,6 +1,7 @@ import json import os from unittest.mock import MagicMock, call, patch +from io import BytesIO from botocore.exceptions import ClientError from requests import Response @@ -11,11 +12,7 @@ from src.main import ( BUCKET_NAME, - create_dictionary, - get_and_update_copilot_teams, get_and_update_historic_usage, - get_copilot_team_date, - get_team_history, handler, update_s3_object, get_dict_value, @@ -58,128 +55,13 @@ def test_update_s3_object_failure(self, caplog): assert any("Failed to update" in record.message for record in caplog.records) -class TestGetAndUpdateCopilotTeams: - @patch("src.main.update_s3_object") - def test_get_and_update_copilot_teams_single_page(self, mock_update_s3_object): - s3 = MagicMock() - gh = MagicMock() - # Mock response for first page - mock_response = MagicMock() - mock_response.links = {} # No 'last' link, so only one page - gh.get.return_value = mock_response - - # Patch get_copilot_team_date to return a list of teams - with patch( - "src.main.get_copilot_team_date", return_value=[{"name": "team1"}] - ) as mock_get_team_date: - result = get_and_update_copilot_teams(s3, gh, False) - assert result == [{"name": "team1"}] - mock_get_team_date.assert_called_once_with(gh, 1) - mock_update_s3_object.assert_called_once() - args, kwargs = mock_update_s3_object.call_args - assert args[1].endswith("copilot-usage-dashboard") - assert args[2] == "copilot_teams.json" - assert args[3] == [{"name": "team1"}] - - @patch("src.main.update_s3_object") - def test_get_and_update_copilot_teams_multiple_pages(self, mock_update_s3_object): - s3 = MagicMock() - gh = MagicMock() - # Mock response with 'last' link for 3 pages - mock_response = MagicMock() - mock_response.links = {"last": {"url": "https://api.github.com/orgs/test/teams?page=3"}} - gh.get.return_value = mock_response - - # Patch get_copilot_team_date to return different teams per page - with patch( - "src.main.get_copilot_team_date", - side_effect=[[{"name": "team1"}], [{"name": "team2"}], [{"name": "team3"}]], - ) as mock_get_team_date: - result = get_and_update_copilot_teams(s3, gh, False) - assert result == [{"name": "team1"}, {"name": "team2"}, {"name": "team3"}] - assert mock_get_team_date.call_count == 3 - mock_update_s3_object.assert_called_once() - - @patch("src.main.update_s3_object") - def test_get_and_update_copilot_teams_no_teams(self, mock_update_s3_object): - s3 = MagicMock() - gh = MagicMock() - mock_response = MagicMock() - mock_response.links = {} - gh.get.return_value = mock_response - - with patch("src.main.get_copilot_team_date", return_value=[]) as mock_get_team_date: - result = get_and_update_copilot_teams(s3, gh, False) - assert result == [] - mock_get_team_date.assert_called_once_with(gh, 1) - mock_update_s3_object.assert_called_once() - args, kwargs = mock_update_s3_object.call_args - assert args[1].endswith("copilot-usage-dashboard") - assert args[2] == "copilot_teams.json" - assert args[3] == [] - - def test_write_data_locally_creates_file(self, tmp_path): - s3 = MagicMock() - gh = MagicMock() - response = MagicMock() - response.links = {} - gh.get.return_value = response - - with patch("src.main.get_copilot_team_date", return_value=[{"name": "teamA"}]): - with patch("src.main.os.makedirs") as mock_makedirs, \ - patch("src.main.open", create=True) as mock_open: - result = get_and_update_copilot_teams(s3, gh, True) - assert result == [{"name": "teamA"}] - mock_makedirs.assert_called_once_with("output", exist_ok=True) - mock_open.assert_called_once() - s3.put_object.assert_not_called() - - -class TestGetTeamHistory: - def setup_method(self): - self.org_patch = patch("src.main.org", "test-org") - self.org_patch.start() - self.addCleanup = getattr(self, "addCleanup", lambda f: None) - - def teardown_method(self): - self.org_patch.stop() - - def test_get_team_history_success(self): - gh = MagicMock() - mock_response = MagicMock(spec=Response) - mock_response.json.return_value = [{"date": "2024-01-01", "usage": 5}] - gh.get.return_value = mock_response - - result = get_team_history(gh, "dev-team", {"since": "2024-01-01"}) - gh.get.assert_called_once_with( - "/orgs/test-org/team/dev-team/copilot/metrics", params={"since": "2024-01-01"} - ) - assert result == [{"date": "2024-01-01", "usage": 5}] - - def test_get_team_history_with_no_query_params(self): - gh = MagicMock() - mock_response = MagicMock(spec=Response) - mock_response.json.return_value = [] - gh.get.return_value = mock_response - - result = get_team_history(gh, "dev-team") - gh.get.assert_called_once_with("/orgs/test-org/team/dev-team/copilot/metrics", params=None) - assert result == [] - - class TestHandler: @patch("src.main.boto3.Session") @patch("src.main.github_api_toolkit.get_token_as_installation") @patch("src.main.github_api_toolkit.github_interface") @patch("src.main.get_and_update_historic_usage") - @patch("src.main.get_and_update_copilot_teams") - @patch("src.main.create_dictionary") - @patch("src.main.update_s3_object") def test_handler_success( self, - mock_update_s3_object, - mock_create_dictionary, - mock_get_and_update_copilot_teams, mock_get_and_update_historic_usage, mock_github_interface, mock_get_token_as_installation, @@ -199,21 +81,10 @@ def test_handler_success( mock_github_interface.return_value = mock_gh mock_get_and_update_historic_usage.return_value = (["usage1", "usage2"], ["2024-01-01"]) - mock_get_and_update_copilot_teams.return_value = [{"name": "team1"}] - mock_create_dictionary.return_value = [ - {"team": {"name": "team1"}, "data": [{"date": "2024-01-01"}]} - ] secret_region = "eu-west-1" secret_name = "test-secret" - # S3 get_object for teams_history.json returns existing history - mock_s3.get_object.return_value = { - "Body": MagicMock( - read=MagicMock(return_value=b'[{"team": {"name": "team1"}, "data": []}]') - ) - } - result = handler({}, MagicMock()) assert result == "Github Data logging is now complete." mock_boto3_session.assert_called_once() @@ -221,13 +92,8 @@ def test_handler_success( call("secretsmanager", region_name=secret_region) in mock_session.client.call_args_list mock_secret_manager.get_secret_value.assert_called_once_with(SecretId=secret_name) mock_get_token_as_installation.assert_called_once() - mock_github_interface.assert_called_once() - mock_get_and_update_historic_usage.assert_called_once() - mock_get_and_update_copilot_teams.assert_called_once() - mock_create_dictionary.assert_called_once() - mock_update_s3_object.assert_called_with( - mock_s3, BUCKET_NAME, "teams_history.json", mock_create_dictionary.return_value - ) + mock_github_interface.assert_called_once_with("token") + mock_get_and_update_historic_usage.assert_called_once_with(mock_s3, mock_gh, False) @patch("src.main.boto3.Session") @patch("src.main.github_api_toolkit.get_token_as_installation") @@ -246,110 +112,6 @@ def test_handler_access_token_error( assert result.startswith("Error getting access token:") assert any("Error getting access token" in record.getMessage() for record in caplog.records) - @patch("src.main.boto3.Session") - @patch("src.main.github_api_toolkit.get_token_as_installation") - @patch("src.main.github_api_toolkit.github_interface") - @patch("src.main.get_and_update_historic_usage") - @patch("src.main.get_and_update_copilot_teams") - @patch("src.main.create_dictionary") - @patch("src.main.update_s3_object") - def test_handler_team_history_client_error( - self, - mock_update_s3_object, - mock_create_dictionary, - mock_get_and_update_copilot_teams, - mock_get_and_update_historic_usage, - mock_github_interface, - mock_get_token_as_installation, - mock_boto3_session, - caplog, - ): - mock_s3 = MagicMock() - mock_secret_manager = MagicMock() - mock_session = MagicMock() - mock_session.client.side_effect = [mock_s3, mock_secret_manager] - mock_boto3_session.return_value = mock_session - - mock_secret_manager.get_secret_value.return_value = {"SecretString": "pem-content"} - mock_get_token_as_installation.return_value = ("token",) - mock_gh = MagicMock() - mock_github_interface.return_value = mock_gh - - mock_get_and_update_historic_usage.return_value = (["usage1"], ["2024-01-01"]) - mock_get_and_update_copilot_teams.return_value = [{"name": "team1"}] - mock_create_dictionary.return_value = [ - {"team": {"name": "team1"}, "data": [{"date": "2024-01-01"}]} - ] - - # S3 get_object for teams_history.json raises ClientError - mock_s3.get_object.side_effect = ClientError( - error_response={"Error": {"Code": "404", "Message": "Not Found"}}, - operation_name="GetObject", - ) - - result = handler({}, MagicMock()) - assert result == "Github Data logging is now complete." - assert any( - "Error retrieving existing team history" in record.getMessage() - for record in caplog.records - ) - mock_update_s3_object.assert_called_with( - mock_s3, BUCKET_NAME, "teams_history.json", mock_create_dictionary.return_value - ) - - -class TestGetCopilotTeamDate: - @patch("src.main.org", "test-org") - def test_get_copilot_team_date_success(self): - gh = MagicMock() - # Mock teams response - teams_response = MagicMock() - teams_response.json.return_value = [ - {"name": "team1", "slug": "slug1", "description": "desc1", "html_url": "url1"}, - {"name": "team2", "slug": "slug2", "description": "desc2", "html_url": "url2"}, - ] - gh.get.side_effect = [ - teams_response, - MagicMock(spec=Response), # usage_data for team1 - MagicMock(spec=Response), # usage_data for team2 - ] - - result = get_copilot_team_date(gh, 1) - assert result == [ - {"name": "team1", "slug": "slug1", "description": "desc1", "url": "url1"}, - {"name": "team2", "slug": "slug2", "description": "desc2", "url": "url2"}, - ] - gh.get.assert_any_call("/orgs/test-org/teams", params={"per_page": 100, "page": 1}) - gh.get.assert_any_call("/orgs/test-org/team/team1/copilot/metrics") - gh.get.assert_any_call("/orgs/test-org/team/team2/copilot/metrics") - - @patch("src.main.org", "test-org") - def test_get_copilot_team_date_unexpected_usage_response(self, caplog): - gh = MagicMock() - teams_response = MagicMock() - teams_response.json.return_value = [ - {"name": "team1", "slug": "slug1", "description": "desc1", "html_url": "url1"}, - ] - gh.get.side_effect = [ - teams_response, - "not_a_response", # usage_data for team1 - ] - - with caplog.at_level("ERROR"): - result = get_copilot_team_date(gh, 1) - assert result == [] - - @patch("src.main.org", "test-org") - def test_get_copilot_team_date_empty_teams(self): - gh = MagicMock() - teams_response = MagicMock() - teams_response.json.return_value = [] - gh.get.return_value = teams_response - - result = get_copilot_team_date(gh, 1) - assert result == [] - gh.get.assert_called_once_with("/orgs/test-org/teams", params={"per_page": 100, "page": 1}) - class TestGetAndUpdateHistoricUsage: def setup_method(self): @@ -362,39 +124,61 @@ def teardown_method(self): def test_get_and_update_historic_usage_success(self): s3 = MagicMock() gh = MagicMock() - # Mock usage data returned from GitHub API - usage_data = [ - {"date": "2024-01-01", "usage": 10}, - {"date": "2024-01-02", "usage": 20}, - ] - gh.get.return_value.json.return_value = usage_data + + # Mock API response + api_response = { + "download_links": [ + "https://example.com/org_history_api_response.json" + ] + # There are other fields in the API response, but we don't need them for this test + } + + # Mock usage data returned from GitHub API + fetched_usage_data = {"day_totals": [ + {"day": "2024-01-01", "usage": 10}, + {"day": "2024-01-02", "usage": 20}, + ]} + + gh.get.return_value.json.return_value = api_response # Mock S3 get_object returns existing historic usage with one date - existing_usage = [{"date": "2024-01-01", "usage": 10}] + existing_usage = [{"day": "2024-01-01", "usage": 10}] s3.get_object.return_value = { - "Body": MagicMock( - read=MagicMock(return_value=json.dumps(existing_usage).encode("utf-8")) - ) + "Body": BytesIO(json.dumps(existing_usage).encode("utf-8")) } - result, dates_added = get_and_update_historic_usage(s3, gh, False) + # Mock requests.get returns usage data from download_links + # We always patch dependencies imported inside the function we're testing. + # Test environment initialisation ends here. + with patch("src.main.requests.get") as mock_requests_get: + mock_requests_get.return_value.json.return_value = fetched_usage_data + result, dates_added = get_and_update_historic_usage(s3, gh, False) + assert result == [ - {"date": "2024-01-01", "usage": 10}, - {"date": "2024-01-02", "usage": 20}, + {"day": "2024-01-01", "usage": 10}, + {"day": "2024-01-02", "usage": 20}, ] assert dates_added == ["2024-01-02"] s3.get_object.assert_called_once() s3.put_object.assert_called_once() args, kwargs = s3.put_object.call_args assert kwargs["Bucket"].endswith("copilot-usage-dashboard") - assert kwargs["Key"] == "historic_usage_data.json" + assert kwargs["Key"] == "org_history.json" assert json.loads(kwargs["Body"].decode("utf-8")) == result def test_get_and_update_historic_usage_no_existing_data(self, caplog): s3 = MagicMock() gh = MagicMock() - usage_data = [{"date": "2024-01-01", "usage": 10}] - gh.get.return_value.json.return_value = usage_data + api_response = { + "download_links": [ + "https://example.com/org_history_api_response.json" + ] + } + fetched_usage_data = {"day_totals": [ + {"day": "2024-01-01", "usage": 10}, + ]} + + gh.get.return_value.json.return_value = api_response # S3 get_object raises ClientError s3.get_object.side_effect = ClientError( @@ -402,39 +186,58 @@ def test_get_and_update_historic_usage_no_existing_data(self, caplog): operation_name="GetObject", ) - result, dates_added = get_and_update_historic_usage(s3, gh, False) - assert result == [{"date": "2024-01-01", "usage": 10}] + with patch("src.main.requests.get") as mock_requests_get: + mock_requests_get.return_value.json.return_value = fetched_usage_data + result, dates_added = get_and_update_historic_usage(s3, gh, False) + + assert result == [{"day": "2024-01-01", "usage": 10}] assert dates_added == ["2024-01-01"] s3.put_object.assert_called_once() assert any( - "Error getting historic_usage_data.json" in record.getMessage() + "Error getting org_history.json" in record.getMessage() for record in caplog.records ) def test_get_and_update_historic_usage_no_new_dates(self): s3 = MagicMock() gh = MagicMock() - usage_data = [{"date": "2024-01-01", "usage": 10}] - gh.get.return_value.json.return_value = usage_data + api_response = { + "download_links": [ + "https://example.com/org_history_api_response.json" + ] + } + fetched_usage_data = {"day_totals": [ + {"day": "2024-01-01", "usage": 10}, + ]} + + gh.get.return_value.json.return_value = api_response # S3 get_object returns same date as usage_data - existing_usage = [{"date": "2024-01-01", "usage": 10}] + existing_usage = [{"day": "2024-01-01", "usage": 10}] s3.get_object.return_value = { - "Body": MagicMock( - read=MagicMock(return_value=json.dumps(existing_usage).encode("utf-8")) - ) + "Body": BytesIO(json.dumps(existing_usage).encode("utf-8")) } + with patch("src.main.requests.get") as mock_requests_get: + mock_requests_get.return_value.json.return_value = fetched_usage_data + result, dates_added = get_and_update_historic_usage(s3, gh, False) - result, dates_added = get_and_update_historic_usage(s3, gh, False) - assert result == [{"date": "2024-01-01", "usage": 10}] + assert result == [{"day": "2024-01-01", "usage": 10}] assert dates_added == [] s3.put_object.assert_called_once() def test_write_data_locally_creates_file(self, tmp_path): s3 = MagicMock() gh = MagicMock() - usage_data = [{"date": "2024-01-01", "usage": 10}] - gh.get.return_value.json.return_value = usage_data + api_response = { + "download_links": [ + "https://example.com/org_history_api_response.json" + ] + } + fetched_usage_data = {"day_totals": [ + {"day": "2024-01-01", "usage": 10}, + ]} + + gh.get.return_value.json.return_value = api_response # S3 get_object raises ClientError s3.get_object.side_effect = ClientError( @@ -444,141 +247,15 @@ def test_write_data_locally_creates_file(self, tmp_path): # Patch os.makedirs and open to use tmp_path with patch("src.main.os.makedirs") as mock_makedirs, \ - patch("src.main.open", create=True) as mock_open: - result, dates_added = get_and_update_historic_usage(s3, gh, True) - assert result == [{"date": "2024-01-01", "usage": 10}] - assert dates_added == ["2024-01-01"] - mock_makedirs.assert_called_once_with("output", exist_ok=True) - mock_open.assert_called_once() - s3.put_object.assert_not_called() - - -class TestCreateDictionary: - def setup_method(self): - self.org_patch = patch("src.main.org", "test-org") - self.org_patch.start() - - def teardown_method(self): - self.org_patch.stop() - - def test_create_dictionary_adds_new_team_history(self): - gh = MagicMock() - copilot_teams = [{"name": "team1"}, {"name": "team2"}] - existing_team_history = [] - - # get_team_history returns history for each team - with patch( - "src.main.get_team_history", - side_effect=[ - [{"date": "2024-01-01", "usage": 5}], - [{"date": "2024-01-02", "usage": 10}], - ], - ) as mock_get_team_history: - result = create_dictionary(gh, copilot_teams, existing_team_history) - assert len(result) == 2 - assert result[0]["team"]["name"] == "team1" - assert result[0]["data"] == [{"date": "2024-01-01", "usage": 5}] - assert result[1]["team"]["name"] == "team2" - assert result[1]["data"] == [{"date": "2024-01-02", "usage": 10}] - assert mock_get_team_history.call_count == 2 - - def test_create_dictionary_extends_existing_team_history(self): - gh = MagicMock() - copilot_teams = [{"name": "team1"}] - existing_team_history = [ - {"team": {"name": "team1"}, "data": [{"date": "2024-01-01", "usage": 5}]} - ] - - # get_team_history returns new history for team1 - with patch( - "src.main.get_team_history", return_value=[{"date": "2024-01-02", "usage": 10}] - ) as mock_get_team_history: - result = create_dictionary(gh, copilot_teams, existing_team_history) - assert len(result) == 1 - assert result[0]["team"]["name"] == "team1" - assert result[0]["data"] == [ - {"date": "2024-01-01", "usage": 5}, - {"date": "2024-01-02", "usage": 10}, - ] - mock_get_team_history.assert_called_once() - - args, kwargs = mock_get_team_history.call_args - assert args[0] == gh - assert args[1] == "team1" - assert args[2] == {"since": "2024-01-01"} - - def test_create_dictionary_skips_team_with_no_name(self, caplog): - gh = MagicMock() - copilot_teams = [{"slug": "slug1"}] # No 'name' - existing_team_history = [] - - with patch("src.main.get_team_history") as mock_get_team_history: - result = create_dictionary(gh, copilot_teams, existing_team_history) - assert result == [] - assert mock_get_team_history.call_count == 0 - assert any( - "Skipping team with no name" in record.getMessage() for record in caplog.records - ) - - def test_create_dictionary_no_new_history(self, caplog): - gh = MagicMock() - copilot_teams = [{"name": "team1"}] - existing_team_history = [] - - # get_team_history returns empty list - with patch("src.main.get_team_history", return_value=[]) as mock_get_team_history: - result = create_dictionary(gh, copilot_teams, existing_team_history) - assert result == [] - assert mock_get_team_history.call_count == 1 - - -class TestGetTeamHistory: - def setup_method(self): - self.org_patch = patch("src.main.org", "test-org") - self.org_patch.start() - - def teardown_method(self): - self.org_patch.stop() - - def test_get_team_history_returns_metrics(self): - gh = MagicMock() - mock_response = MagicMock(spec=Response) - mock_response.json.return_value = [{"date": "2024-01-01", "usage": 5}] - gh.get.return_value = mock_response - - result = get_team_history(gh, "dev-team", {"since": "2024-01-01"}) - gh.get.assert_called_once_with( - "/orgs/test-org/team/dev-team/copilot/metrics", params={"since": "2024-01-01"} - ) - assert result == [{"date": "2024-01-01", "usage": 5}] - - def test_get_team_history_returns_empty_list(self): - gh = MagicMock() - mock_response = MagicMock(spec=Response) - mock_response.json.return_value = [] - gh.get.return_value = mock_response - - result = get_team_history(gh, "dev-team") - gh.get.assert_called_once_with("/orgs/test-org/team/dev-team/copilot/metrics", params=None) - assert result == [] - - def test_get_team_history_non_response_returns_none(self): - gh = MagicMock() - gh.get.return_value = "not_a_response" - - result = get_team_history(gh, "dev-team") - gh.get.assert_called_once_with("/orgs/test-org/team/dev-team/copilot/metrics", params=None) - assert result is None - - def test_get_team_history_with_query_params_none(self): - gh = MagicMock() - mock_response = MagicMock(spec=Response) - mock_response.json.return_value = [{"date": "2024-01-01", "usage": 5}] - gh.get.return_value = mock_response - - result = get_team_history(gh, "dev-team", None) - gh.get.assert_called_once_with("/orgs/test-org/team/dev-team/copilot/metrics", params=None) - assert result == [{"date": "2024-01-01", "usage": 5}] + patch("src.main.open", create=True) as mock_open, \ + patch("src.main.requests.get") as mock_requests_get: + mock_requests_get.return_value.json.return_value = fetched_usage_data + result, dates_added = get_and_update_historic_usage(s3, gh, True) + assert result == [{"day": "2024-01-01", "usage": 10}] + assert dates_added == ["2024-01-01"] + mock_makedirs.assert_called_once_with("output", exist_ok=True) + mock_open.assert_called_once() + s3.put_object.assert_not_called() class TestGetDictValue: