AutoRemove Pending nodes

[sidmore]

Is your feature request related to a problem? Please describe.
Operator queries CCF to get pending nodes and this list can keep growing in the event the Governor gets throttled for proposals that get a 503 for removing the node, resulting in a perpetual growing list.

Logs for such a case:
all_with_backoff", "ModuleName": "ccfcommonutils.ccforchclients._client", "CorrelationId": null, "Message": "[GET] https://calastone-ccf-pilot3:443/node/network/nodes - Params: {}", "Exception": null} {"LineType": "Trace", "Timestamp": "2024-10-02 16:07:36.131224+00:00", "Level": "INFO", "FileName": "_timed_activity.py", "LineNo": 160, "Function": "end", "ModuleName": "ccfcommonutils.utils.timer._timed_activity", "CorrelationId": null, "Message": "TimedActivity complete: {\"activityType\": \"ApiCall\", \"description\": \"An HTTP request\", \"elapsedTimeInSeconds\": 0.43145087599987164, \"name\": \"[GET] https://calastone-ccf-pilot3:443/node/network/nodes\", \"state\": \"Succeeded\"}", "Exception": null} {"LineType": "Trace", "Timestamp": "2024-10-02 16:07:36.131412+00:00", "Level": "INFO", "FileName": "_client.py", "LineNo": 253, "Function": "call_with_backoff", "ModuleName": "ccfcommonutils.ccforchclients._client", "CorrelationId": null, "Message": "[GET] https://calastone-ccf-pilot3:443/node/network/nodes - Status: 200", "Exception": null} {"LineType": "Trace", "Timestamp": "2024-10-02 16:07:38.203612+00:00", "Level": "INFO", "FileName": "instance.py", "LineNo": 254, "Function": "_get_duplicates_from_ccf", "ModuleName": "ccforchestrator.initializer.instance", "CorrelationId": null, "Message": "Found NodeStatus.PENDING duplicate d854559630fb02a34d6773a29ba5a4d5da25d06c34701a942d6a9eabb119ef2a with name ccf-4", "Exception": null} {"LineType": "Trace", "Timestamp": "2024-10-02 16:07:38.203882+00:00", "Level": "INFO", "FileName": "instance.py", "LineNo": 254, "Function": "_get_duplicates_from_ccf", "ModuleName": "ccforchestrator.initializer.instance", "CorrelationId": null, "Message": "Found NodeStatus.PENDING duplicate 9d43ee6d046703aed387df855f80b8d3b5d32b48fc0d53fedd1af26e823000d3 with name ccf-4", "Exception": null} {"LineType": "Trace", "Timestamp": "2024-10-02 16:07:38.204144+00:00", "Level": "INFO", "FileName": "instance.py", "LineNo": 254, "Function": "_get_duplicates_from_ccf", "ModuleName": "ccforchestrator.initializer.instance", "CorrelationId": null, "Message": "Found NodeStatus.PENDING duplicate 96dbad35612de9dada5643b59af54ba6481bf0e6d5833c484052f0546a970371 with name ccf-4", "Exception": null} {"LineType": "Trace", "Timestamp": "2024-10-02 16:07:38.204341+00:00", "Level": "INFO", "FileName": "instance.py", "LineNo": 254, "Function": "_get_duplicates_from_ccf", "ModuleName": "ccforchestrator.initializer.instance", "CorrelationId": null, "Message": "Found NodeStatus.PENDING duplicate fb64a6fa58e70b77f50f6b88618a94658f78997f365ca3e1c856a4e1516d3472 with name ccf-4", "Exception": null} {"LineType": "Trace", "Timestamp": "2024-10-02 16:07:38.204462+00:00", "Level": "INFO", "FileName": "instance.py", "LineNo": 254, "Function": "_get_duplicates_from_ccf", "ModuleName": "ccforchestrator.initializer.instance", "CorrelationId": null, "Message": "Found NodeStatus.PENDING duplicate 2482776c016c08a379ef1fd85bf5675ef42ce7a2b6543c11b0eccfaadc5ecb6e with name ccf-4", "Exception": null} {"LineType": "Trace", "Timestamp": "2024-10-02 16:07:38.204568+00:00", "Level": "INFO", "FileName": "instance.py", "LineNo": 254, "Function": "_get_duplicates_from_ccf", "ModuleName": "ccforchestrator.initializer.instance", "CorrelationId": null, "Message": "Found NodeStatus.PENDING duplicate fbfb7a0804728e8cbeb55618e06616f480b610fb5885f9c1101788f9eb6ef67a with name ccf-4", "Exception": null} {"LineType": "Trace", "Timestamp": "2024-10-02 16:07:38.204675+00:00", "Level": "INFO", "FileName": "instance.py", "LineNo": 254, "Function": "_get_duplicates_from_ccf", "ModuleName": "ccforchestrator.initializer.instance", "CorrelationId": null, "Message": "Found NodeStatus.PENDING duplicate a793e1fa673a762e68e685a999caaa145cca815e5333470517580acac0f5ca6f with name ccf-4", "Exception": null} {"LineType": "Trace", "Timestamp": "2024-10-02 16:07:38.204792+00:00", "Level": "INFO", "FileName": "instance.py", "LineNo": 254, "Function": "_get_duplicates_from_ccf", "ModuleName": "ccforchestrator.initializer.instance", "CorrelationId": null, "Message": "Found NodeStatus.PENDING duplicate 47adbb9e0e03a906a47ecf0e7861833563e8dafbb8f2e53eaf60981f01f14922 with name ccf-4", "Exception": null} {"LineType": "Trace", "Timestamp": "2024-10-02 16:07:38.204909+00:00", "Level": "INFO", "FileName": "instance.py", "LineNo": 254, "Function": "_get_duplicates_from_ccf", "ModuleName": "ccforchestrator.initializer.instance", "CorrelationId": null, "Message": "Found NodeStatus.PENDING duplicate 854328978cb18b3fc95e6ae4585ee0c614d2360e2208ba8bc8a9d641dc417844 with name ccf-4", "Exception": null} {"LineType": "Trace", "Timestamp": "2024-10-02 16:07:38.205024+00:00", "Level": "INFO", "FileName": "instance.py", "LineNo": 254, "Function": "_get_duplicates_from_ccf", "ModuleName": "ccforchestrator.initializer.instance", "CorrelationId": null, "Message": "Found NodeStatus.PENDING duplicate fbe5f05386ff0330713ddc69546a3f66a655f37a92a65665e01da7bb3b5d7321 with name ccf-4", "Exception": null} {"LineType": "Trace", "Timestamp": "2024-10-02 16:07:38.205128+00:00", "Level": "INFO", "FileName": "instance.py", "LineNo": 254, "Function": "_get_duplicates_from_ccf", "ModuleName": "ccforchestrator.initializer.instance", "CorrelationId": null, "Message": "Found NodeStatus.PENDING duplicate 7be4e93481acfea00d435c493f98bc9278dde67cec0cc9b844c881cb4eac3cbc with name ccf-4", "Exception": null} {"LineType": "Trace", "Timestamp": "2024-10-02 16:07:38.205244+00:00", "Level": "INFO", "FileName": "instance.py", "LineNo": 254, "Function": "_get_duplicates_from_ccf", "ModuleName": "ccforchestrator.initializer.instance", "CorrelationId": null, "Message": "Found NodeStatus.PENDING duplicate 29526c305a65d03d50a3638551afab9a11e2035b34d01a30af8e8b9c70fedd95 with name ccf-4", "Exception": null} {"LineType": "Trace", "Timestamp": "2024-10-02 16:07:38.205347+00:00", "Level": "INFO", "FileName": "instance.py", "LineNo": 254, "Function": "_get_duplicates_from_ccf", "ModuleName": "ccforchestrator.initializer.instance", "CorrelationId": null, "Message": "Found NodeStatus.PENDING duplicate 7a478defee89a1975c4480d30a72e2fd00c5a33e0e1e5faba2738c38f8abe916 with name ccf-4", "Exception": null} {"LineType": "Trace", "Timestamp": "2024-10-02 16:07:38.205471+00:00", "Level": "INFO", "FileName": "instance.py", "LineNo": 254, "Function": "_get_duplicates_from_ccf", "ModuleName": "ccforchestrator.initializer.instance", "CorrelationId": null, "Message": "Found NodeStatus.PENDING duplicate 358f8f23eab6ca57abd5400cf5c2830904a3f1b0c2b5daee96dfda03e20c0c5d with name ccf-4", "Exception": null} {"LineType": "Trace", "Timestamp": "2024-10-02 16:07:38.205593+00:00", "Level": "INFO", "FileName": "instance.py", "LineNo": 254, "Function": "_get_duplicates_from_ccf", "ModuleName": "ccforchestrator.initializer.instance", "CorrelationId": null, "Message": "Found NodeStatus.PENDING duplicate 57e2e0c240482ba49b4282dadc9d8f305129bc19823c5ef771675c98a986f6cd with name ccf-4", "Exception": null} {"LineType": "Trace", "Timestamp": "2024-10-02 16:07:38.205717+00:00", "Level": "INFO", "FileName": "instance.py", "LineNo": 254, "Function": "_get_duplicates_from_ccf", "ModuleName": "ccforchestrator.initializer.instance", "CorrelationId": null, "Message": "Found NodeStatus.PENDING duplicate 018de29149cb57f62dfe01c0f39ba7aba5da7f4bd44af6d746740908bd85db5a with name ccf-4", "Exception": null}
Describe the solution you'd like
CCF can keep a timer and auto remove these nodes, simplifying Operator responsibilities having to remove nodes.

Describe alternatives you've considered
A clear and concise description of any alternative solutions or features you've considered.

Additional context
Add any other context about the feature request here.