refactor: ota_proxy: use simple-sqlite3-orm instead of vendoring an o…

…rm inside otaproxy package (#363) This PR introduces to use simple-sqlite3-orm package(which is also implemented by me) instead. This package is feature-rich, well-implemented and well-tested replacement for the vendored sqlite3 ORM inside the otaproxy package. Also this PR fixes a potential race condition, which the database entry being committed before the cache file being finalized, resulting small chance of cache file not found on database lookup hit. Other major changes * split cache_streaming related logic intocache_streaming module. * split lru_cache_helper from ota_cache module into lru_cache_helper module. * in lru_cache_helper, now we limit the max steps to walk down the bucket list when rotating the cache. * in db, now we use RETURNING statement when rotating cache on supported platform. * in ota_cache, now we also check db integrity to determine whether to force init.
tier4 · Aug 2, 2024 · 4ee5c29 · 4ee5c29 · github-actions · Aug 2, 2024
1 parent 1915c1c
commit 4ee5c29
Show file tree

Hide file tree

Showing 13 changed files with 942 additions and 1,435 deletions.
diff --git a/pyproject.toml b/pyproject.toml
@@ -34,6 +34,7 @@ dependencies = [
   "pyopenssl<25,>=24.1",
   "pyyaml<7,>=6.0.1",
   "requests<2.33,>=2.32",
+  "simple-sqlite3-orm @ https://github.com/pga2rn/simple-sqlite3-orm/releases/download/v0.2.0/simple_sqlite3_orm-0.2.0-py3-none-any.whl",
   "typing-extensions>=4.6.3",
   "urllib3<2.3,>=2.2.2",
   "uvicorn[standard]<0.31,>=0.30",
@@ -45,7 +46,7 @@ optional-dependencies.dev = [
   "flake8",
   "isort",
   "pytest==7.1.2",
-  "pytest-asyncio==0.21",
+  "pytest-asyncio==0.23.8",
   "pytest-mock==3.14",
   "requests-mock",
 ]
@@ -54,6 +55,9 @@ urls.Source = "https://github.com/tier4/ota-client"
 [tool.hatch.version]
 source = "vcs"
 
+[tool.hatch.metadata]
+allow-direct-references = true
+
 [tool.hatch.build.hooks.vcs]
 version-file = "src/_otaclient_version.py"
 

diff --git a/src/ota_proxy/__init__.py b/src/ota_proxy/__init__.py
@@ -24,7 +24,7 @@
 
 from typing_extensions import ParamSpec, Self
 
-from .cache_control import OTAFileCacheControl
+from .cache_control_header import OTAFileCacheControl
 from .config import config
 from .ota_cache import OTACache
 from .server_app import App

diff --git a/src/ota_proxy/_consts.py b/src/ota_proxy/_consts.py
@@ -1,6 +1,6 @@
 from multidict import istr
 
-from .cache_control import OTAFileCacheControl
+from .cache_control_header import OTAFileCacheControl
 
 # uvicorn
 REQ_TYPE_LIFESPAN = "lifespan"

diff --git a/src/ota_proxy/cache_control.py → src/ota_proxy/cache_control_header.py b/src/ota_proxy/cache_control.py → src/ota_proxy/cache_control_header.py
File	Stmts	Miss	Cover	Missing
src/ota_metadata/legacy
__init__.py	11	0	100%
parser.py	335	38	88%	100, 156, 161, 197–198, 208–209, 212, 224, 282, 292–295, 334–337, 417, 420, 428–430, 443, 452–453, 456–457, 669–670, 673, 700–702, 752, 755–757
types.py	84	13	84%	37, 40–42, 112–116, 122–125
src/ota_proxy
__init__.py	36	10	72%	59, 61, 63, 72, 81–82, 102, 104–106
__main__.py	7	7	0%	16–18, 20, 22–23, 25
_consts.py	15	0	100%
cache_control_header.py	68	4	94%	71, 91, 113, 121
cache_streaming.py	184	25	86%	170, 181, 208–210, 230, 246–247, 249, 272–273, 279, 283, 285, 312–314, 330, 391–392, 438, 451–454
config.py	20	0	100%
db.py	73	15	79%	109, 115, 173, 180–181, 192–199, 201–202
errors.py	5	0	100%
lru_cache_helper.py	47	2	95%	84–85
ota_cache.py	214	59	72%	69–70, 139, 150–151, 183–184, 201, 238–242, 246–248, 250, 252–259, 261–263, 266–267, 271–272, 276, 323, 331–333, 412–415, 428, 431–432, 446–447, 449–451, 455–456, 462–463, 494, 500, 527, 556–558
server_app.py	138	39	71%	76, 79, 85, 101, 103, 162, 171, 213–214, 216–218, 221, 226–228, 231–232, 235, 238, 241, 244, 257–258, 261–262, 264, 267, 293–296, 299, 313–315, 321–323
utils.py	23	1	95%	33
src/otaclient
__init__.py	5	2	60%	17, 19
__main__.py	1	1	0%	16
log_setting.py	52	5	90%	53, 55, 64–66
src/otaclient/app
__main__.py	1	1	0%	16
configs.py	76	0	100%
errors.py	120	0	100%
interface.py	5	0	100%
main.py	46	5	89%	52–53, 75–77
ota_client.py	373	108	71%	76, 84, 105, 132, 134–135, 137, 141, 145–146, 151–152, 158, 160, 198–201, 207, 211, 217, 336, 348–349, 351, 360, 363, 368–369, 372, 378, 380–384, 403–406, 409–416, 444–447, 493–494, 498, 500–501, 531–532, 541–548, 555, 558–564, 609–612, 620, 656–658, 663–665, 668–669, 671–672, 674, 732–733, 736, 744–745, 748, 759–760, 763, 771–772, 775, 786, 805, 832, 851, 869
ota_client_stub.py	394	109	72%	76–78, 80–81, 89–92, 95–97, 101, 106–107, 109–110, 113, 115–116, 119–121, 124–125, 128–130, 135–140, 144, 147–151, 153–154, 162–164, 167, 204–206, 211, 247, 272, 275, 278, 382, 406, 408, 432, 478, 535, 605–606, 645, 664–666, 672–675, 679–681, 688–690, 693, 697–700, 753, 842–844, 851, 881–882, 885–889, 898–907, 914, 920, 923–924, 928, 931
update_stats.py	104	9	91%	57, 103, 105, 114, 116, 125, 127, 148, 179
src/otaclient/app/boot_control
__init__.py	4	0	100%
_common.py	248	112	54%	74–75, 96–98, 114–115, 135–136, 155–156, 175–176, 195–196, 218–220, 235–236, 260–266, 287, 295, 313, 321, 340–341, 344–345, 368, 370–379, 381–390, 392–394, 413, 416, 424, 432, 448–450, 452–457, 550, 555, 560, 673, 677–678, 681, 689, 691–692, 718–719, 721–724, 729, 735–736, 739–740, 742, 749–750, 761–767, 775–777, 781–782, 785–786, 789, 795
_grub.py	417	128	69%	216, 264–267, 273–277, 314–315, 322–327, 330–336, 339, 342–343, 348, 350–352, 361–367, 369–370, 372–374, 383–385, 387–389, 468–469, 473–474, 526, 532, 558, 580, 584–585, 600–602, 626–629, 641, 645–647, 649–651, 710–713, 738–741, 764–767, 779–780, 783–784, 819, 825, 845–846, 848, 860, 863, 866, 869, 873–875, 893–896, 924–927, 932–940, 945–953
_jetson_cboot.py	270	214	20%	69–70, 77–78, 96–105, 117, 124–125, 137, 143–144, 154–156, 168–169, 180–181, 184–185, 188–189, 192–196, 199–200, 204–205, 210–211, 213–217, 219–225, 227–228, 233, 236, 239–240, 243, 247–248, 252–253, 257, 260, 263, 267–273, 275–277, 282, 285, 288, 292, 299, 301–304, 317, 320, 324, 326–328, 332, 339, 341, 344, 350–351, 356, 364, 372–374, 383–384, 386–388, 394, 397–399, 403–404, 406, 409, 418–420, 423, 426, 429–434, 436–438, 441, 444, 448–453, 457–459, 464–465, 469–470, 473, 476, 479–480, 483, 486, 491, 494, 497–498, 500, 502, 505, 508, 510–511, 514–518, 523–524, 526, 534–538, 540, 543, 546, 557–558, 563, 573, 576–584, 589–597, 602–610, 616–618, 621, 624
_jetson_common.py	141	66	53%	50, 74, 129–134, 136, 141–143, 148–151, 159–160, 167–168, 173–174, 190–191, 193–195, 198–200, 203, 207, 211, 215–217, 223–224, 226, 259, 285–286, 288–290, 294–297, 299–300, 302–306, 308, 315–316, 319, 321, 331, 334–335, 338, 340
_rpi_boot.py	286	134	53%	54, 57, 121–122, 126, 134–137, 151–154, 161–162, 164–165, 170–171, 174–175, 184–185, 223, 229–233, 236, 254–256, 260–262, 267–269, 273–275, 285–286, 289, 292, 294–295, 297–298, 300–302, 308, 311–312, 322–325, 333–337, 339, 341–342, 347–348, 355–361, 392, 394–397, 407–410, 414–415, 417–421, 449–452, 471–474, 479, 482, 500–503, 508–516, 521–529, 546–549, 555–557, 560, 563
configs.py	38	0	100%
protocol.py	4	0	100%
selecter.py	38	26	31%	44–46, 49–50, 54–55, 58–60, 63, 65, 69, 77–79, 81–82, 84–85, 89, 91–93, 95, 97
src/otaclient/app/create_standby
__init__.py	12	5	58%	28–30, 32, 34
common.py	224	44	80%	59, 62–63, 67–69, 71, 75–76, 78, 124, 172–174, 176–178, 180, 183–186, 190, 201, 275–276, 278–283, 295, 332, 360, 363–365, 381–382, 396, 400, 422–423
interface.py	5	0	100%
rebuild_mode.py	97	9	90%	89–91, 103–108
src/otaclient/configs
_common.py	8	0	100%
ecu_info.py	57	1	98%	107
proxy_info.py	52	2	96%	88, 90
src/otaclient_api/v2
__init__.py	14	0	100%
api_caller.py	39	6	84%	45–47, 83–85
api_stub.py	17	0	100%
types.py	256	23	91%	86, 89–92, 131, 209–210, 212, 259, 262–263, 506–508, 512–513, 515, 518–519, 522–523, 586
src/otaclient_common
__init__.py	34	8	76%	42–44, 59, 61, 67, 74–75
common.py	154	19	87%	41, 45, 200, 203–205, 220, 227–229, 295–297, 307, 316–318, 364, 368
downloader.py	200	10	95%	110–111, 129, 156, 372, 426, 430, 518–519, 528
linux.py	61	15	75%	51–53, 59, 69, 74, 76, 108–109, 133–134, 190, 195–196, 198
logging.py	29	1	96%	55
persist_file_handling.py	118	18	84%	113, 118, 150–152, 163, 192–193, 228–232, 242–244, 246–247
proto_streamer.py	42	8	80%	33, 48, 66–67, 72, 81–82, 100
proto_wrapper.py	398	48	87%	87, 165, 172, 184–186, 205, 210, 221, 257, 263, 268, 299, 303, 307, 402, 462, 469, 472, 492, 499, 501, 526, 532, 535, 537, 562, 568, 571, 573, 605, 609, 611, 625, 642, 669, 672, 676, 707, 713, 760–763, 765, 803–805
retry_task_map.py	80	7	91%	164–165, 167, 179–182
typing.py	25	3	88%	69–70, 72
TOTAL	5805	1360	76%