|
From time to time, some tests fail with AWS_ERROR_PRIORITY_QUEUE_EMPTY.
For example,
__________ TestCpWithCRTClient.test_streaming_upload_using_crt_client __________
[gw26] linux -- Python 3.11.8 /build/aws-cli-v2/src/awscli-2.15.19/venv/bin/python
self = <tests.functional.s3.test_cp_command.TestCpWithCRTClient testMethod=test_streaming_upload_using_crt_client>
def test_streaming_upload_using_crt_client(self):
cmdline = [
's3', 'cp', '-', 's3://bucket/key'
]
with mock.patch('sys.stdin', BufferedBytesIO(b'foo')):
> self.run_command(cmdline)
tests/functional/s3/test_cp_command.py:2124:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
tests/functional/s3/__init__.py:424: in run_command
self.assertEqual(
E AssertionError: 255 != 0 : Expected rc of 0 instead got 255 with stderr message:
E 30 (AWS_ERROR_PRIORITY_QUEUE_EMPTY): Attempt to pop an item from an empty queue.
------------------------------ Captured log call -------------------------------
DEBUG awscli.clidriver:clidriver.py:466 Exception caught in main()
Traceback (most recent call last):
File "/build/aws-cli-v2/src/awscli-2.15.19/awscli/clidriver.py", line 460, in main
return command_table[parsed_args.command](remaining, parsed_args)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/build/aws-cli-v2/src/awscli-2.15.19/awscli/customizations/commands.py", line 151, in __call__
return self._subcommand_table[subcommand_name](
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/build/aws-cli-v2/src/awscli-2.15.19/awscli/customizations/commands.py", line 205, in __call__
rc = self._run_main(parsed_args, parsed_globals)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/build/aws-cli-v2/src/awscli-2.15.19/awscli/customizations/s3/subcommands.py", line 717, in _run_main
transfer_manager = self._get_transfer_manager(
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/build/aws-cli-v2/src/awscli-2.15.19/awscli/customizations/s3/subcommands.py", line 759, in _get_transfer_manager
return TransferManagerFactory(self._session).create_transfer_manager(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/build/aws-cli-v2/src/awscli-2.15.19/awscli/customizations/s3/factory.py", line 69, in create_transfer_manager
return self._create_crt_transfer_manager(params, runtime_config)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/build/aws-cli-v2/src/awscli-2.15.19/awscli/customizations/s3/factory.py", line 116, in _create_crt_transfer_manager
self._create_crt_client(params, runtime_config),
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/build/aws-cli-v2/src/awscli-2.15.19/awscli/customizations/s3/factory.py", line 139, in _create_crt_client
return create_s3_crt_client(**create_crt_client_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/build/aws-cli-v2/src/awscli-2.15.19/awscli/s3transfer/crt.py", line 133, in create_s3_crt_client
event_loop_group = EventLoopGroup(num_threads)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.11/site-packages/awscrt/io.py", line 87, in __init__
self._binding = _awscrt.event_loop_group_new(num_threads, is_pinned, cpu_group, on_shutdown)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
RuntimeError: 30 (AWS_ERROR_PRIORITY_QUEUE_EMPTY): Attempt to pop an item from an empty queue.
With the following debugging patch for aws-cli:
diff --git a/tests/functional/s3/test_cp_command.py b/tests/functional/s3/test_cp_command.py
index 2864621bd..abc51d326 100644
--- a/tests/functional/s3/test_cp_command.py
+++ b/tests/functional/s3/test_cp_command.py
@@ -2029,7 +2029,7 @@ class TestCpWithCRTClient(BaseCRTTransferClientTest):
def test_upload_using_crt_client(self):
filename = self.files.create_file('myfile', 'mycontent')
cmdline = [
- 's3', 'cp', filename, 's3://bucket/key'
+ 's3', '--debug', 'cp', filename, 's3://bucket/key'
]
self.run_command(cmdline)
crt_requests = self.get_crt_make_request_calls()
@@ -2046,7 +2046,7 @@ class TestCpWithCRTClient(BaseCRTTransferClientTest):
filename1 = self.files.create_file('myfile1', 'mycontent')
filename2 = self.files.create_file('myfile2', 'mycontent')
cmdline = [
- 's3', 'cp', self.files.rootdir, 's3://bucket/', '--recursive'
+ 's3', '--debug', 'cp', self.files.rootdir, 's3://bucket/', '--recursive'
]
self.run_command(cmdline)
crt_requests = self.get_crt_make_request_calls()
@@ -2069,7 +2069,7 @@ class TestCpWithCRTClient(BaseCRTTransferClientTest):
def test_download_using_crt_client(self):
filename = os.path.join(self.files.rootdir, 'myfile')
cmdline = [
- 's3', 'cp', 's3://bucket/key', filename
+ 's3', '--debug', 'cp', 's3://bucket/key', filename
]
self.add_botocore_head_object_response()
self.run_command(cmdline)
@@ -2085,7 +2085,7 @@ class TestCpWithCRTClient(BaseCRTTransferClientTest):
def test_recursive_download_using_crt_client(self):
cmdline = [
- 's3', 'cp', 's3://bucket/', self.files.rootdir, '--recursive'
+ 's3', '--debug', 'cp', 's3://bucket/', self.files.rootdir, '--recursive'
]
self.add_botocore_list_objects_response(['key1', 'key2'])
self.run_command(cmdline)
@@ -2108,7 +2108,7 @@ class TestCpWithCRTClient(BaseCRTTransferClientTest):
def test_does_not_use_crt_client_for_copies(self):
cmdline = [
- 's3', 'cp', 's3://bucket/key', 's3://otherbucket/'
+ 's3', '--debug', 'cp', 's3://bucket/key', 's3://otherbucket/'
]
self.add_botocore_head_object_response()
self.add_botocore_copy_object_response()
@@ -2118,7 +2118,7 @@ class TestCpWithCRTClient(BaseCRTTransferClientTest):
def test_streaming_upload_using_crt_client(self):
cmdline = [
- 's3', 'cp', '-', 's3://bucket/key'
+ 's3', '--debug', 'cp', '-', 's3://bucket/key'
]
with mock.patch('sys.stdin', BufferedBytesIO(b'foo')):
self.run_command(cmdline)
@@ -2134,7 +2134,7 @@ class TestCpWithCRTClient(BaseCRTTransferClientTest):
def test_streaming_download_using_crt_client(self):
cmdline = [
- 's3', 'cp', 's3://bucket/key', '-'
+ 's3', '--debug', 'cp', 's3://bucket/key', '-'
]
result = self.run_command(cmdline)
crt_requests = self.get_crt_make_request_calls()
@@ -2152,7 +2152,7 @@ class TestCpWithCRTClient(BaseCRTTransferClientTest):
def test_respects_region_parameter(self):
filename = self.files.create_file('myfile', 'mycontent')
cmdline = [
- 's3', 'cp', filename, 's3://bucket/key', '--region', 'us-west-1',
+ 's3', '--debug', 'cp', filename, 's3://bucket/key', '--region', 'us-west-1',
]
self.run_command(cmdline)
self.assert_crt_client_region('us-west-1')
@@ -2169,7 +2169,7 @@ class TestCpWithCRTClient(BaseCRTTransferClientTest):
def test_respects_endpoint_url_parameter(self):
filename = self.files.create_file('myfile', 'mycontent')
cmdline = [
- 's3', 'cp', filename, 's3://bucket/key',
+ 's3', '--debug', 'cp', filename, 's3://bucket/key',
'--endpoint-url', 'https://my.endpoint.com'
]
self.run_command(cmdline)
@@ -2190,7 +2190,7 @@ class TestCpWithCRTClient(BaseCRTTransferClientTest):
def test_can_disable_ssl_using_endpoint_url_parameter(self):
filename = self.files.create_file('myfile', 'mycontent')
cmdline = [
- 's3', 'cp', filename, 's3://bucket/key',
+ 's3', '--debug', 'cp', filename, 's3://bucket/key',
'--endpoint-url', 'http://my.endpoint.com'
]
self.run_command(cmdline)
@@ -2211,7 +2211,7 @@ class TestCpWithCRTClient(BaseCRTTransferClientTest):
def test_respects_no_sign_request_parameter(self):
filename = self.files.create_file('myfile', 'mycontent')
cmdline = [
- 's3', 'cp', filename, 's3://bucket/key', '--no-sign-request'
+ 's3', '--debug', 'cp', filename, 's3://bucket/key', '--no-sign-request'
]
self.run_command(cmdline)
self.assert_crt_client_has_no_credential_provider()
@@ -2230,7 +2230,7 @@ class TestCpWithCRTClient(BaseCRTTransferClientTest):
fake_ca_contents = b"fake ca content"
ca_bundle = self.files.create_file('fake_ca', fake_ca_contents, mode='wb')
cmdline = [
- 's3', 'cp', filename, 's3://bucket/key', '--ca-bundle', ca_bundle
+ 's3', '--debug', 'cp', filename, 's3://bucket/key', '--ca-bundle', ca_bundle
]
self.run_command(cmdline)
crt_requests = self.get_crt_make_request_calls()
@@ -2243,7 +2243,7 @@ class TestCpWithCRTClient(BaseCRTTransferClientTest):
filename = self.files.create_file('myfile', 'mycontent')
ca_bundle = self.files.create_file('fake_ca', 'mycontent')
cmdline = [
- 's3', 'cp', filename, 's3://bucket/key', '--ca-bundle', ca_bundle, '--no-verify-ssl'
+ 's3', '--debug', 'cp', filename, 's3://bucket/key', '--ca-bundle', ca_bundle, '--no-verify-ssl'
]
self.run_command(cmdline)
crt_requests = self.get_crt_make_request_calls()
diff --git a/tests/functional/s3/test_mv_command.py b/tests/functional/s3/test_mv_command.py
index 4170bad36..12adda406 100644
--- a/tests/functional/s3/test_mv_command.py
+++ b/tests/functional/s3/test_mv_command.py
@@ -249,7 +249,7 @@ class TestMvWithCRTClient(BaseCRTTransferClientTest):
def test_upload_move_using_crt_client(self):
filename = self.files.create_file('myfile', 'mycontent')
cmdline = [
- 's3', 'mv', filename, 's3://bucket/key',
+ 's3', '--debug', 'mv', filename, 's3://bucket/key',
]
self.run_command(cmdline)
crt_requests = self.get_crt_make_request_calls()
@@ -266,7 +266,7 @@ class TestMvWithCRTClient(BaseCRTTransferClientTest):
def test_download_move_using_crt_client(self):
filename = os.path.join(self.files.rootdir, 'myfile')
cmdline = [
- 's3', 'mv', 's3://bucket/key', filename
+ 's3', '--debug', 'mv', 's3://bucket/key', filename
]
self.add_botocore_head_object_response()
self.add_botocore_delete_object_response()
@@ -285,7 +285,7 @@ class TestMvWithCRTClient(BaseCRTTransferClientTest):
def test_does_not_use_crt_client_for_copy_moves(self):
cmdline = [
- 's3', 'mv', 's3://bucket/key', 's3://otherbucket/'
+ 's3', '--debug', 'mv', 's3://bucket/key', 's3://otherbucket/'
]
self.add_botocore_head_object_response()
self.add_botocore_copy_object_response()
diff --git a/tests/functional/s3/test_rm_command.py b/tests/functional/s3/test_rm_command.py
index 3af1d6c48..02b774630 100644
--- a/tests/functional/s3/test_rm_command.py
+++ b/tests/functional/s3/test_rm_command.py
@@ -75,7 +75,7 @@ class TestRmCommand(BaseS3TransferCommandTest):
class TestRmWithCRTClient(BaseCRTTransferClientTest):
def test_delete_using_crt_client(self):
cmdline = [
- 's3', 'rm', 's3://bucket/key'
+ 's3', '--debug', 'rm', 's3://bucket/key'
]
self.run_command(cmdline)
crt_requests = self.get_crt_make_request_calls()
@@ -90,7 +90,7 @@ class TestRmWithCRTClient(BaseCRTTransferClientTest):
def test_recursive_delete_using_crt_client(self):
cmdline = [
- 's3', 'rm', 's3://bucket/', '--recursive'
+ 's3', '--debug', 'rm', 's3://bucket/', '--recursive'
]
self.add_botocore_list_objects_response(['key1', 'key2'])
self.run_command(cmdline)
diff --git a/tests/functional/s3/test_sync_command.py b/tests/functional/s3/test_sync_command.py
index 6d3cf6625..7f1782870 100644
--- a/tests/functional/s3/test_sync_command.py
+++ b/tests/functional/s3/test_sync_command.py
@@ -385,7 +385,7 @@ class TestSyncWithCRTClient(BaseCRTTransferClientTest):
def test_upload_sync_using_crt_client(self):
filename = self.files.create_file('myfile', 'mycontent')
cmdline = [
- 's3', 'sync', self.files.rootdir, 's3://bucket/',
+ 's3', '--debug', 'sync', self.files.rootdir, 's3://bucket/',
]
self.add_botocore_list_objects_response([])
self.run_command(cmdline)
@@ -401,7 +401,7 @@ class TestSyncWithCRTClient(BaseCRTTransferClientTest):
def test_download_sync_using_crt_client(self):
cmdline = [
- 's3', 'sync', 's3://bucket/', self.files.rootdir,
+ 's3', '--debug', 'sync', 's3://bucket/', self.files.rootdir,
]
self.add_botocore_list_objects_response(['key'])
self.run_command(cmdline)
@@ -418,7 +418,7 @@ class TestSyncWithCRTClient(BaseCRTTransferClientTest):
def test_upload_sync_with_delete_using_crt_client(self):
filename = self.files.create_file('a-file', 'mycontent')
cmdline = [
- 's3', 'sync', self.files.rootdir, 's3://bucket/', '--delete'
+ 's3', '--debug', 'sync', self.files.rootdir, 's3://bucket/', '--delete'
]
self.add_botocore_list_objects_response(['delete-this'])
self.run_command(cmdline)
@@ -442,7 +442,7 @@ class TestSyncWithCRTClient(BaseCRTTransferClientTest):
def test_download_sync_with_delete_using_crt_client(self):
self.files.create_file('delete-this', 'content')
cmdline = [
- 's3', 'sync', 's3://bucket/', self.files.rootdir, '--delete'
+ 's3', '--debug', 'sync', 's3://bucket/', self.files.rootdir, '--delete'
]
self.add_botocore_list_objects_response(['key'])
self.run_command(cmdline)
@@ -459,7 +459,7 @@ class TestSyncWithCRTClient(BaseCRTTransferClientTest):
def test_does_not_use_crt_client_for_copy_syncs(self):
cmdline = [
- 's3', 'sync', 's3://bucket/', 's3://otherbucket/'
+ 's3', '--debug', 'sync', 's3://bucket/', 's3://otherbucket/'
]
self.add_botocore_list_objects_response(['key'])
self.add_botocore_list_objects_response([])
And the following debugging patch for aws-c-io submodule in python-awscrt:
diff --git a/source/linux/epoll_event_loop.c b/source/linux/epoll_event_loop.c
index 094a7836..86e4a8a9 100644
--- a/source/linux/epoll_event_loop.c
+++ b/source/linux/epoll_event_loop.c
@@ -149,7 +149,8 @@ struct aws_event_loop *aws_event_loop_new_default_with_options(
epoll_loop->epoll_fd = epoll_create(100);
if (epoll_loop->epoll_fd < 0) {
- AWS_LOGF_FATAL(AWS_LS_IO_EVENT_LOOP, "id=%p: Failed to open epoll handle.", (void *)loop);
+ int error_code = errno;
+ AWS_LOGF_FATAL(AWS_LS_IO_EVENT_LOOP, "id=%p: Failed to open epoll handle. error=%d", (void *)loop, error_code);
aws_raise_error(AWS_ERROR_SYS_CALL_FAILURE);
goto clean_up_epoll;
}
I can see this in test logs:
[FATAL] [2024-04-02T16:41:36Z] [000070a056e2b740] [event-loop] - id=0x62604dc4af80: Failed to open epoll handle. error=24
I guess the error 24 (EMFILE, Too many open files) is caused by too high
parallelism, so I raise the ulimit as a workaround. It seems the error
never happens again with the new ulimit.
|