182 lines
5.6 KiB
JSON
182 lines
5.6 KiB
JSON
{
|
|
"head": "HEL-889 shutdown ownership & queued-cancel",
|
|
"real_llm_requests": 0,
|
|
"real_embeddings": 0,
|
|
"boundary": "deterministic 512 vectors, non-loopback forbidden",
|
|
"checks": [
|
|
{
|
|
"name": "Q1 queued snapshot shows phantom-liveness",
|
|
"ok": true,
|
|
"detail": ""
|
|
},
|
|
{
|
|
"name": "Q1 cancel releases registry and requeues",
|
|
"ok": true,
|
|
"detail": "{\"registered_workers\": 0, \"batch\": {\"state\": \"queued\", \"reason\": \"queued_worker_cancelled\", \"owner\": null, \"attempts\": 0, \"entry_ids\": null}, \"row_done\": 0, \"stored_entries\": 0}"
|
|
},
|
|
{
|
|
"name": "Q1 in-process recovery completes without restart",
|
|
"ok": true,
|
|
"detail": "{\"batch\": {\"state\": \"done\", \"reason\": null, \"owner\": null, \"attempts\": 1, \"entry_ids\": \"[\\\"stub-1\\\"]\"}, \"registered_workers\": 0, \"stored_entries\": 1, \"row_done\": 1}"
|
|
},
|
|
{
|
|
"name": "S1 aclose does NOT return while the worker is alive",
|
|
"ok": true,
|
|
"detail": ""
|
|
},
|
|
{
|
|
"name": "S1 worker completes and flushes before store close",
|
|
"ok": true,
|
|
"detail": "{\"state\": \"done\", \"reason\": null, \"owner\": null, \"attempts\": 1, \"entry_ids\": \"[\\\"19039615-6f32-48e6-979b-715d2cd56fcb\\\"]\"}"
|
|
},
|
|
{
|
|
"name": "S1 no late-worker error after honest drain",
|
|
"ok": true,
|
|
"detail": ""
|
|
},
|
|
{
|
|
"name": "S1 durable reopen: done + row_done=1 + llm_used=1",
|
|
"ok": true,
|
|
"detail": "{\"state\": \"done\", \"reason\": null, \"owner\": null, \"attempts\": 1, \"entry_ids\": \"[\\\"19039615-6f32-48e6-979b-715d2cd56fcb\\\"]\", \"row_done\": 1, \"llm_used\": 1, \"gateway_posts\": 1}"
|
|
},
|
|
{
|
|
"name": "S2 shutdown between rows requeues with first row proven",
|
|
"ok": true,
|
|
"detail": "{\"state\": \"queued\", \"reason\": \"shutdown_between_rows\", \"owner\": null, \"attempts\": 0, \"entry_ids\": null, \"row_done\": 1, \"gateway_posts_during_shutdown\": 1}"
|
|
},
|
|
{
|
|
"name": "S2 resume sends ONLY the missing row and completes",
|
|
"ok": true,
|
|
"detail": "{\"state\": \"done\", \"reason\": null, \"owner\": null, \"attempts\": 1, \"entry_ids\": \"[\\\"f26d374a-858e-4dde-b2de-ae8af7057021\\\", \\\"258ab9a5-d523-45b0-ada1-d58fbda50ea7\\\"]\", \"row_done\": 2, \"gateway_posts_resume\": 2}"
|
|
},
|
|
{
|
|
"name": "S3 cancelled aclose closes nothing and re-arms",
|
|
"ok": true,
|
|
"detail": ""
|
|
},
|
|
{
|
|
"name": "S3 worker finishes done and a later aclose closes for real",
|
|
"ok": true,
|
|
"detail": ""
|
|
},
|
|
{
|
|
"name": "S4 concurrent aclose drains once and closes once",
|
|
"ok": true,
|
|
"detail": "{\"batch\": {\"state\": \"done\", \"reason\": null, \"owner\": null, \"attempts\": 1, \"entry_ids\": \"[\\\"e553422c-1840-4ff4-b51c-a3586cc34179\\\"]\", \"row_done\": 1}, \"client_usable_after_close\": false}"
|
|
}
|
|
],
|
|
"queued_cancel": {
|
|
"queued": {
|
|
"registered_workers": 1,
|
|
"product_executor_queue": 1
|
|
},
|
|
"after_cancel": {
|
|
"registered_workers": 0,
|
|
"batch": {
|
|
"state": "queued",
|
|
"reason": "queued_worker_cancelled",
|
|
"owner": null,
|
|
"attempts": 0,
|
|
"entry_ids": null
|
|
},
|
|
"row_done": 0,
|
|
"stored_entries": 0
|
|
},
|
|
"after_recovery": {
|
|
"batch": {
|
|
"state": "done",
|
|
"reason": null,
|
|
"owner": null,
|
|
"attempts": 1,
|
|
"entry_ids": "[\"stub-1\"]"
|
|
},
|
|
"registered_workers": 0,
|
|
"stored_entries": 1,
|
|
"row_done": 1
|
|
}
|
|
},
|
|
"shutdown_timeout": {
|
|
"deadline": {
|
|
"aclose_returned_while_worker_alive": false,
|
|
"batch_before_any_close": {
|
|
"state": "pending",
|
|
"reason": null,
|
|
"owner": "4a048d364194a961365c30ee1e570233",
|
|
"attempts": 1,
|
|
"entry_ids": null
|
|
},
|
|
"client_usable": true
|
|
},
|
|
"before_store_close": {
|
|
"state": "done",
|
|
"reason": null,
|
|
"owner": null,
|
|
"attempts": 1,
|
|
"entry_ids": "[\"19039615-6f32-48e6-979b-715d2cd56fcb\"]"
|
|
},
|
|
"late_worker_error": null,
|
|
"persisted": {
|
|
"state": "done",
|
|
"reason": null,
|
|
"owner": null,
|
|
"attempts": 1,
|
|
"entry_ids": "[\"19039615-6f32-48e6-979b-715d2cd56fcb\"]",
|
|
"row_done": 1,
|
|
"llm_used": 1,
|
|
"gateway_posts": 1
|
|
}
|
|
},
|
|
"between_rows": {
|
|
"after_shutdown": {
|
|
"state": "queued",
|
|
"reason": "shutdown_between_rows",
|
|
"owner": null,
|
|
"attempts": 0,
|
|
"entry_ids": null,
|
|
"row_done": 1,
|
|
"gateway_posts_during_shutdown": 1
|
|
},
|
|
"after_resume": {
|
|
"state": "done",
|
|
"reason": null,
|
|
"owner": null,
|
|
"attempts": 1,
|
|
"entry_ids": "[\"f26d374a-858e-4dde-b2de-ae8af7057021\", \"258ab9a5-d523-45b0-ada1-d58fbda50ea7\"]",
|
|
"row_done": 2,
|
|
"gateway_posts_resume": 2
|
|
}
|
|
},
|
|
"cancelled_aclose": {
|
|
"after_cancel": {
|
|
"client_usable": true,
|
|
"batch": {
|
|
"state": "pending",
|
|
"reason": null,
|
|
"owner": "a712e0367aa123b265b47e2d3223846a",
|
|
"attempts": 1,
|
|
"entry_ids": null
|
|
},
|
|
"closing_cleared": true
|
|
},
|
|
"after_worker": {
|
|
"state": "done",
|
|
"reason": null,
|
|
"owner": null,
|
|
"attempts": 1,
|
|
"entry_ids": "[\"73f87468-f546-46d7-8116-37d4fe2818b1\"]"
|
|
},
|
|
"client_usable_after_real_close": false
|
|
},
|
|
"concurrent_aclose": {
|
|
"batch": {
|
|
"state": "done",
|
|
"reason": null,
|
|
"owner": null,
|
|
"attempts": 1,
|
|
"entry_ids": "[\"e553422c-1840-4ff4-b51c-a3586cc34179\"]",
|
|
"row_done": 1
|
|
},
|
|
"client_usable_after_close": false
|
|
},
|
|
"failures": []
|
|
} |