mirror of
https://github.com/helm/chartmuseum.git
synced 2026-02-05 15:45:50 +01:00
0e9d2e36ea
Index regeneration before this PR was causing a concurrency pile-up when many requests were coming in at a fast pace. This was causing the time to return /index to always increase. The behavior was observed on a high latency connection to Google Cloud Storage, but would also occur with other network stores. With this PR, the regeneration time is constant and thus the serving of index.yaml is also constant. The implementation takes a general approach to first make initial network requests (to satisfy the object list from storage) and to pile up subsequent requests while the initial one is being completed. The same algorithm is used to update the in-memory cached index. Note that these requests need not be protected by a lock because they are only ever executing on their own. See server.getChartList() for the implementaion of this idea. A refactor was needed to separate the fetch from the diff calculation to allow separate calling of the network heavy operations. While doing so, we also removed redundant calls to storage file list update. Also made small low-hanging fruit style optimisations to index manipulations. Added request ID to all requests for better debugging. This will be visible with the --debug flag. This was indispensable to diagnose complex concurrent processing. To test the before and after state, we have added the use of the locusti.io loadtesting engine. A simple README in the loadtesting/ directory shows how to install locust (with pipenv) and loadtest chartmuseum. This will prove useful in the future. Fixes #18
225 lines
12 KiB
JSON
225 lines
12 KiB
JSON
{
|
|
"_meta": {
|
|
"hash": {
|
|
"sha256": "61aaaf047ac37f113fb2f160addc46622690eca395ca67d7ef029f596b45cfd6"
|
|
},
|
|
"host-environment-markers": {
|
|
"implementation_name": "cpython",
|
|
"implementation_version": "0",
|
|
"os_name": "posix",
|
|
"platform_machine": "x86_64",
|
|
"platform_python_implementation": "CPython",
|
|
"platform_release": "15.6.0",
|
|
"platform_system": "Darwin",
|
|
"platform_version": "Darwin Kernel Version 15.6.0: Sun Jun 4 21:43:07 PDT 2017; root:xnu-3248.70.3~1/RELEASE_X86_64",
|
|
"python_full_version": "2.7.13",
|
|
"python_version": "2.7",
|
|
"sys_platform": "darwin"
|
|
},
|
|
"pipfile-spec": 6,
|
|
"requires": {},
|
|
"sources": [
|
|
{
|
|
"name": "pypi",
|
|
"url": "https://pypi.python.org/simple",
|
|
"verify_ssl": true
|
|
}
|
|
]
|
|
},
|
|
"default": {
|
|
"certifi": {
|
|
"hashes": [
|
|
"sha256:244be0d93b71e93fc0a0a479862051414d0e00e16435707e5bf5000f92e04694",
|
|
"sha256:5ec74291ca1136b40f0379e1128ff80e866597e4e2c1e755739a913bbc3613c0"
|
|
],
|
|
"version": "==2017.11.5"
|
|
},
|
|
"chardet": {
|
|
"hashes": [
|
|
"sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691",
|
|
"sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae"
|
|
],
|
|
"version": "==3.0.4"
|
|
},
|
|
"click": {
|
|
"hashes": [
|
|
"sha256:29f99fc6125fbc931b758dc053b3114e55c77a6e4c6c3a2674a2dc986016381d",
|
|
"sha256:f15516df478d5a56180fbf80e68f206010e6d160fc39fa508b65e035fd75130b"
|
|
],
|
|
"version": "==6.7"
|
|
},
|
|
"flask": {
|
|
"hashes": [
|
|
"sha256:0749df235e3ff61ac108f69ac178c9770caeaccad2509cb762ce1f65570a8856",
|
|
"sha256:49f44461237b69ecd901cc7ce66feea0319b9158743dd27a2899962ab214dac1"
|
|
],
|
|
"version": "==0.12.2"
|
|
},
|
|
"gevent": {
|
|
"hashes": [
|
|
"sha256:9b492bb1a043540abb6e54fdb5537531e24962ca49c09f3b47dc4f9c37f6297c",
|
|
"sha256:de13a8e378103af84a8bf6015ad1d2761d46f29b8393e8dd6d9bb7cb51bbb713",
|
|
"sha256:deafd70d04ab62428d4e291e8e2c0fb22f38690e6a9f23a67ee6c304087634da",
|
|
"sha256:b67a10799923f9fed546ca5f8b93a2819c71a60132d7a97b4a13fbdab66b278a",
|
|
"sha256:35790f1a3c8e431ada3471b70bb2105050009ea4beb15cbe41b86bc716a7ffa9",
|
|
"sha256:c9dd6534c46ed782e2d7236767cd07115cb29ce8670c2fc0794f264de9024fe0",
|
|
"sha256:b7e0e6400c2f3ce78a9ae1cdd55b53166feedd003d60c033863881227129a4d3",
|
|
"sha256:0901975628790e8a57fc92bb7062e5b856edea48c8de9caf36cfda14eae07329",
|
|
"sha256:df52e06a2754c2d905aad75a7dc06a732c804d9edbc87f06f47c8f483ba98bca",
|
|
"sha256:70558dd45c7a1f8046ba45792e489dd0f409bd8a3b7a0635ca9d3055223b3dff",
|
|
"sha256:8a710eddb3e9e5f22bdbd458b5f211b94f59409ecd6896f15b9fee2cba266a59",
|
|
"sha256:60109741377367eef8ded9283a1bf629621b73acaf3e1e8aac9d1a0f50fa0f05",
|
|
"sha256:552719cec4721673b8c7d2f9de666e3f7591b9b182f801ecaef1c76e638052aa",
|
|
"sha256:a16db4f56699ef07f0249b953ff949aae641e50b2bdc4710f11c0d8d9089b296",
|
|
"sha256:59e9237af027f8db85e5d78a9da2e328ae96f01d67a0d62abcecad3db7876908",
|
|
"sha256:833bebdc36bfeeedefc200ca9aee9b8eddd80f56b63ca1e886e18b97b1240edd",
|
|
"sha256:81cb24e0f7bd9888596364e8d8ed0d65c2547c84884c67bb46d956faeed67396",
|
|
"sha256:1af93825db5753550fa8ff5ab2f2132e8733170b3f8d38347b34fa4a984cb624",
|
|
"sha256:2ff045a91509c35664c27a849c8cbf742a227f587b7cdbc88301e9c85dcaedff",
|
|
"sha256:a66cf99f08da65c501826a19e30f5a6e7ba942fdd79baba5ce2d51eebaa13444",
|
|
"sha256:4791c8ae9c57d6f153354736e1ccab1e2baf6c8d9ae5a77a9ac90f41e2966b2d",
|
|
"sha256:74bce0c30bb2240e3d5d515ba8cb3eadf840c2bde7109a1979c7a26c9d0f5a6a",
|
|
"sha256:a0ed8ba787b9c0c1c565c2675d71652e6c1e2d4e91f53530860d0303e867fe85",
|
|
"sha256:c35b29de49211014ec66d056fd4f9ba7a04795e2a654697f72879c0cf365d6d4",
|
|
"sha256:6892fabc9051e8c0a171d543b6536859aabeb6d169db79b2f45d64dc2a15808c",
|
|
"sha256:fce894a64db3911897cdad6c37fbb23dfb18b7bf8b9cb8c00a8ea0a7253651c9",
|
|
"sha256:4f098002126ebef7f2907188b6c8b09e5193161ce968847d9e6a8bc832b0db9a",
|
|
"sha256:33fa6759eabc9176ddbe0d29b66867a82e19a61f06eb7cfabbac35343c0ecf24",
|
|
"sha256:7f93b67b680f4a921f517294048d05f8f6f0ed5962b78d6685a6cf0fcd7d8202"
|
|
],
|
|
"version": "==1.2.2"
|
|
},
|
|
"greenlet": {
|
|
"hashes": [
|
|
"sha256:96888e47898a471073b394ea641b7d675c1d054c580dd4a04a382bd34e67d89e",
|
|
"sha256:d2d5103f6cba131e1be660230018e21f276911d2b68b629ead1c5cb5e5472ac7",
|
|
"sha256:bc339de0e0969de5118d0b62a080a7611e2ba729a90f4a3ad78559c51bc5576d",
|
|
"sha256:b8ab98f8ae25938326dc4c21e3689a933531500ae4f3bfcefe36e3e25fda4dbf",
|
|
"sha256:416a3328d7e0a19aa1df3ec09524a109061fd7b80e010ef0dff9f695b4ac5e20",
|
|
"sha256:21232907c8c26838b16915bd8fbbf82fc70c996073464cc70981dd4a96bc841c",
|
|
"sha256:6803d8c6b235c861c50afddf00c7467ffbcd5ab960d137ff0f9c36f2cb11ee4b",
|
|
"sha256:76dab055476dd4dabb00a967b4df1990b25542d17eaa40a18f66971d10193e0b",
|
|
"sha256:70b9ff28921f5a3c03df4896ec8c55f5f94c593d7a79abd98b4c5c4a692ba873",
|
|
"sha256:7114b757b4146f4c87a0f00f1e58abd4c4729836679af0fc37266910a4a72eb0",
|
|
"sha256:0d90c709355ed13f16676f84e5a9cd67826a9f5c5143381c21e8fc3100ade1f1",
|
|
"sha256:ebae83b6247f83b1e8d887733dfa8046ce6e29d8b3e2a7380256e9de5c6ae55d",
|
|
"sha256:e841e3ece633acae5e2bf6102140a605ffee7d5d4921dca1625c5fdc0f0b3248",
|
|
"sha256:3e5e9be157ece49e4f97f3225460caf758ccb00f934fcbc5db34367cc1ff0aee",
|
|
"sha256:e77b708c37b652c7501b9f8f6056b23633c567aaa0d29edfef1c11673c64b949",
|
|
"sha256:0da1fc809c3bdb93fbacd0f921f461aacd53e554a7b7d4e9953ba09131c4206e",
|
|
"sha256:66fa5b101fcf4521138c1a29668074268d938bbb7de739c8faa9f92ea1f05e1f",
|
|
"sha256:e5451e1ce06b74a4861576c2db74405a4398c4809a105774550a9e52cfc8c4da",
|
|
"sha256:9c407aa6adfd4eea1232e81aa9f3cb3d9b955a9891c4819bf9b498c77efba14b",
|
|
"sha256:b56ac981f07b77e72ad5154278b93396d706572ea52c2fce79fee2abfcc8bfa6",
|
|
"sha256:e4c99c6010a5d153d481fdaf63b8a0782825c0721506d880403a3b9b82ae347e"
|
|
],
|
|
"version": "==0.4.12"
|
|
},
|
|
"idna": {
|
|
"hashes": [
|
|
"sha256:8c7309c718f94b3a625cb648ace320157ad16ff131ae0af362c9f21b80ef6ec4",
|
|
"sha256:2c6a5de3089009e3da7c5dde64a141dbc8551d5b7f6cf4ed7c2568d0cc520a8f"
|
|
],
|
|
"version": "==2.6"
|
|
},
|
|
"itsdangerous": {
|
|
"hashes": [
|
|
"sha256:cbb3fcf8d3e33df861709ecaf89d9e6629cff0a217bc2848f1b41cd30d360519"
|
|
],
|
|
"version": "==0.24"
|
|
},
|
|
"jinja2": {
|
|
"hashes": [
|
|
"sha256:74c935a1b8bb9a3947c50a54766a969d4846290e1e788ea44c1392163723c3bd",
|
|
"sha256:f84be1bb0040caca4cea721fcbbbbd61f9be9464ca236387158b0feea01914a4"
|
|
],
|
|
"version": "==2.10"
|
|
},
|
|
"locustio": {
|
|
"hashes": [
|
|
"sha256:64583987ba1c330bb071aee3e29d2eedbfb7c8b342fa064bfb74fafcff660d61"
|
|
],
|
|
"version": "==0.8.1"
|
|
},
|
|
"markupsafe": {
|
|
"hashes": [
|
|
"sha256:a6be69091dac236ea9c6bc7d012beab42010fa914c459791d627dad4910eb665"
|
|
],
|
|
"version": "==1.0"
|
|
},
|
|
"msgpack-python": {
|
|
"hashes": [
|
|
"sha256:637b012c9ea021de7a7a75d6ff5e82cfef6694babd7e14bb9a3adcb2a5bd52f0",
|
|
"sha256:658c1cd5dcf7786e0e7a6d523cd0c5b33f92e139e224bd73cb3a23ada618d2dc",
|
|
"sha256:920bbbaee07ad048a4d2b4160901b19775c61ef9439f856c74509e763a326249",
|
|
"sha256:e165006f7e3d2612f1bffe2f6f042ca317d8df724d8b72a39b14c2e46c67eaae",
|
|
"sha256:95d70edd50e3d2f6ea1189f77190e4a0172626e7405ddd1689f3f64814447cba",
|
|
"sha256:7e1b12ea0134460052fabcfaa0f488ec0fc21deb14832d66236fd2870757d8f1",
|
|
"sha256:8f36890251f20d96267618cf64735759d7ef7e91bc0b86b9480547d2d1397a68",
|
|
"sha256:1e68a277e4180baa7789be36f27f0891660205f6209f78a32282d3c422873d78",
|
|
"sha256:f52d9f96df952369fe4adcb0506e10c1c92d47f653f601a66da2a26a7e7141ea",
|
|
"sha256:58c9c1d7891a35bddc6ee5dbec10d347a7ae4983169c24fc5fc8a57ae792ca76",
|
|
"sha256:1a2b19df0f03519ec7f19f826afb935b202d8979b0856c6fb3dc28955799f886"
|
|
],
|
|
"version": "==0.4.8"
|
|
},
|
|
"pyzmq": {
|
|
"hashes": [
|
|
"sha256:f88d8ffa32bf729bed806ca3205846ed98b61f149562dc828db5c3a2194ca0c3",
|
|
"sha256:83c30a7fa401d5f26378414c6e67bdf8633780cf514b31bd2084b9ab4b226484",
|
|
"sha256:5f357871b560bfacfa62a737292c59b42fe06c6e3433567f49f42dec32b45b11",
|
|
"sha256:6602d9958656a255a067ab87b569695a51e275bb77111b057c12ac702544fdca",
|
|
"sha256:068977e0f5cfba05d910df9899bcbd379e6536666ddabb2ce6a1218673229c65",
|
|
"sha256:29fe1d6a79ea3b4cfcb3b23ef2159b34a42a347f74010fe5510484543a5367e2",
|
|
"sha256:09ecf151c912f1d0907bfda331fa0ed5cd0032bf1d8733512f1befc8e89b387f",
|
|
"sha256:edde3830b2ac4c97f11fdafeea02dffd2343b470482864a3cd3894c839a0a416",
|
|
"sha256:37d7c8bfa3c7e8121ca7a71fe807cd5b63793d05ad598df53449d49e5dea1cc3",
|
|
"sha256:9848772e952f2addbb77182ae14ef201fd4f0e394e6e73c4620b9fd8f154f4a9",
|
|
"sha256:4d2bda85a2fe827fe1cce007f4bab20dae4bf56fa468d19bc730bb225c349bf3",
|
|
"sha256:7ded1d19978e52557348fd89f2c3f67eb73c19700b248355bcc50ac582387a95",
|
|
"sha256:b3c6abbebd79866465d6bbc0ab239cc4d1ba3bcce203b5294bb494665dcf4105",
|
|
"sha256:c382cb87d65de9718c738ee3a2f4f7c8444bd5d1a002bd370b5365d906c13f27",
|
|
"sha256:f41626459031c36457a6af56dd91e432b315b0bce8458ba08edbf6e4de88b467",
|
|
"sha256:afe36a0ab8ada3b8d9a005c6aaca8a434c425bd019b7fa6d323b9fda46157389",
|
|
"sha256:686ffacf9c62bdd4253ab3e093b32aaf195961c7545583a416cc534caa8a5900",
|
|
"sha256:09802528d0092b70131a9df92c975bb625cf2038fc0f53915bf51ffa665a7957",
|
|
"sha256:22ea664f4d115cd9fa0f2dc8c907298437f4d2fcf0677c18eddf45da2436f923",
|
|
"sha256:505d2a4fc2a899d30663e12550dca3d17d48c5796dd1307a0197656bc40b67c8",
|
|
"sha256:657147db62a16794aca7a18b51139180bc98d6249931507b1c8ef696607ada43",
|
|
"sha256:3a942e278f79ddcf455ba160881a95de474123667c3c847683bc3ae1929093f5",
|
|
"sha256:8a883824147523c0fe76d247dd58994c1c28ef07f1cc5dde595a4fd1c28f2580"
|
|
],
|
|
"version": "==16.0.3"
|
|
},
|
|
"requests": {
|
|
"hashes": [
|
|
"sha256:6a1b267aa90cac58ac3a765d067950e7dbbf75b1da07e895d1f594193a40a38b",
|
|
"sha256:9c443e7324ba5b85070c4a818ade28bfabedf16ea10206da1132edaa6dda237e"
|
|
],
|
|
"version": "==2.18.4"
|
|
},
|
|
"six": {
|
|
"hashes": [
|
|
"sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb",
|
|
"sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9"
|
|
],
|
|
"version": "==1.11.0"
|
|
},
|
|
"urllib3": {
|
|
"hashes": [
|
|
"sha256:06330f386d6e4b195fbfc736b297f58c5a892e4440e54d294d7004e3a9bbea1b",
|
|
"sha256:cc44da8e1145637334317feebd728bd869a35285b93cbb4cca2577da7e62db4f"
|
|
],
|
|
"version": "==1.22"
|
|
},
|
|
"werkzeug": {
|
|
"hashes": [
|
|
"sha256:e8549c143af3ce6559699a01e26fa4174f4c591dbee0a499f3cd4c3781cdec3d",
|
|
"sha256:903a7b87b74635244548b30d30db4c8947fe64c5198f58899ddcd3a13c23bb26"
|
|
],
|
|
"version": "==0.12.2"
|
|
}
|
|
},
|
|
"develop": {}
|
|
}
|