Avoid intermittent test failures

From time to time, some tests fail with AWS_ERROR_PRIORITY_QUEUE_EMPTY.
For example,

__________ TestCpWithCRTClient.test_streaming_upload_using_crt_client __________
[gw26] linux -- Python 3.11.8 /build/aws-cli-v2/src/awscli-2.15.19/venv/bin/python

self = <tests.functional.s3.test_cp_command.TestCpWithCRTClient testMethod=test_streaming_upload_using_crt_client>

    def test_streaming_upload_using_crt_client(self):
        cmdline = [
            's3', 'cp', '-', 's3://bucket/key'
        ]
        with mock.patch('sys.stdin', BufferedBytesIO(b'foo')):
>           self.run_command(cmdline)

tests/functional/s3/test_cp_command.py:2124:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
tests/functional/s3/__init__.py:424: in run_command
    self.assertEqual(
E   AssertionError: 255 != 0 : Expected rc of 0 instead got 255 with stderr message:
E   30 (AWS_ERROR_PRIORITY_QUEUE_EMPTY): Attempt to pop an item from an empty queue.
------------------------------ Captured log call -------------------------------
DEBUG    awscli.clidriver:clidriver.py:466 Exception caught in main()
Traceback (most recent call last):
  File "/build/aws-cli-v2/src/awscli-2.15.19/awscli/clidriver.py", line 460, in main
    return command_table[parsed_args.command](remaining, parsed_args)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/build/aws-cli-v2/src/awscli-2.15.19/awscli/customizations/commands.py", line 151, in __call__
    return self._subcommand_table[subcommand_name](
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/build/aws-cli-v2/src/awscli-2.15.19/awscli/customizations/commands.py", line 205, in __call__
    rc = self._run_main(parsed_args, parsed_globals)
         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/build/aws-cli-v2/src/awscli-2.15.19/awscli/customizations/s3/subcommands.py", line 717, in _run_main
    transfer_manager = self._get_transfer_manager(
                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/build/aws-cli-v2/src/awscli-2.15.19/awscli/customizations/s3/subcommands.py", line 759, in _get_transfer_manager
    return TransferManagerFactory(self._session).create_transfer_manager(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/build/aws-cli-v2/src/awscli-2.15.19/awscli/customizations/s3/factory.py", line 69, in create_transfer_manager
    return self._create_crt_transfer_manager(params, runtime_config)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/build/aws-cli-v2/src/awscli-2.15.19/awscli/customizations/s3/factory.py", line 116, in _create_crt_transfer_manager
    self._create_crt_client(params, runtime_config),
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/build/aws-cli-v2/src/awscli-2.15.19/awscli/customizations/s3/factory.py", line 139, in _create_crt_client
    return create_s3_crt_client(**create_crt_client_kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/build/aws-cli-v2/src/awscli-2.15.19/awscli/s3transfer/crt.py", line 133, in create_s3_crt_client
    event_loop_group = EventLoopGroup(num_threads)
                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3.11/site-packages/awscrt/io.py", line 87, in __init__
    self._binding = _awscrt.event_loop_group_new(num_threads, is_pinned, cpu_group, on_shutdown)
                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
RuntimeError: 30 (AWS_ERROR_PRIORITY_QUEUE_EMPTY): Attempt to pop an item from an empty queue.

With the following debugging patch for aws-cli:

diff --git a/tests/functional/s3/test_cp_command.py b/tests/functional/s3/test_cp_command.py
index 2864621bd..abc51d326 100644
--- a/tests/functional/s3/test_cp_command.py
+++ b/tests/functional/s3/test_cp_command.py
@@ -2029,7 +2029,7 @@ class TestCpWithCRTClient(BaseCRTTransferClientTest):
     def test_upload_using_crt_client(self):
         filename = self.files.create_file('myfile', 'mycontent')
         cmdline = [
-            's3', 'cp', filename, 's3://bucket/key'
+            's3', '--debug', 'cp', filename, 's3://bucket/key'
         ]
         self.run_command(cmdline)
         crt_requests = self.get_crt_make_request_calls()
@@ -2046,7 +2046,7 @@ class TestCpWithCRTClient(BaseCRTTransferClientTest):
         filename1 = self.files.create_file('myfile1', 'mycontent')
         filename2 = self.files.create_file('myfile2', 'mycontent')
         cmdline = [
-            's3', 'cp', self.files.rootdir, 's3://bucket/', '--recursive'
+            's3', '--debug', 'cp', self.files.rootdir, 's3://bucket/', '--recursive'
         ]
         self.run_command(cmdline)
         crt_requests = self.get_crt_make_request_calls()
@@ -2069,7 +2069,7 @@ class TestCpWithCRTClient(BaseCRTTransferClientTest):
     def test_download_using_crt_client(self):
         filename = os.path.join(self.files.rootdir, 'myfile')
         cmdline = [
-            's3', 'cp', 's3://bucket/key', filename
+            's3', '--debug', 'cp', 's3://bucket/key', filename
         ]
         self.add_botocore_head_object_response()
         self.run_command(cmdline)
@@ -2085,7 +2085,7 @@ class TestCpWithCRTClient(BaseCRTTransferClientTest):

     def test_recursive_download_using_crt_client(self):
         cmdline = [
-            's3', 'cp', 's3://bucket/', self.files.rootdir, '--recursive'
+            's3', '--debug', 'cp', 's3://bucket/', self.files.rootdir, '--recursive'
         ]
         self.add_botocore_list_objects_response(['key1', 'key2'])
         self.run_command(cmdline)
@@ -2108,7 +2108,7 @@ class TestCpWithCRTClient(BaseCRTTransferClientTest):

     def test_does_not_use_crt_client_for_copies(self):
         cmdline = [
-            's3', 'cp', 's3://bucket/key', 's3://otherbucket/'
+            's3', '--debug', 'cp', 's3://bucket/key', 's3://otherbucket/'
         ]
         self.add_botocore_head_object_response()
         self.add_botocore_copy_object_response()
@@ -2118,7 +2118,7 @@ class TestCpWithCRTClient(BaseCRTTransferClientTest):

     def test_streaming_upload_using_crt_client(self):
         cmdline = [
-            's3', 'cp', '-', 's3://bucket/key'
+            's3', '--debug', 'cp', '-', 's3://bucket/key'
         ]
         with mock.patch('sys.stdin', BufferedBytesIO(b'foo')):
             self.run_command(cmdline)
@@ -2134,7 +2134,7 @@ class TestCpWithCRTClient(BaseCRTTransferClientTest):

     def test_streaming_download_using_crt_client(self):
         cmdline = [
-            's3', 'cp', 's3://bucket/key', '-'
+            's3', '--debug', 'cp', 's3://bucket/key', '-'
         ]
         result = self.run_command(cmdline)
         crt_requests = self.get_crt_make_request_calls()
@@ -2152,7 +2152,7 @@ class TestCpWithCRTClient(BaseCRTTransferClientTest):
     def test_respects_region_parameter(self):
         filename = self.files.create_file('myfile', 'mycontent')
         cmdline = [
-            's3', 'cp', filename, 's3://bucket/key', '--region', 'us-west-1',
+            's3', '--debug', 'cp', filename, 's3://bucket/key', '--region', 'us-west-1',
         ]
         self.run_command(cmdline)
         self.assert_crt_client_region('us-west-1')
@@ -2169,7 +2169,7 @@ class TestCpWithCRTClient(BaseCRTTransferClientTest):
     def test_respects_endpoint_url_parameter(self):
         filename = self.files.create_file('myfile', 'mycontent')
         cmdline = [
-            's3', 'cp', filename, 's3://bucket/key',
+            's3', '--debug', 'cp', filename, 's3://bucket/key',
             '--endpoint-url', 'https://my.endpoint.com'
         ]
         self.run_command(cmdline)
@@ -2190,7 +2190,7 @@ class TestCpWithCRTClient(BaseCRTTransferClientTest):
     def test_can_disable_ssl_using_endpoint_url_parameter(self):
         filename = self.files.create_file('myfile', 'mycontent')
         cmdline = [
-            's3', 'cp', filename, 's3://bucket/key',
+            's3', '--debug', 'cp', filename, 's3://bucket/key',
             '--endpoint-url', 'http://my.endpoint.com'
         ]
         self.run_command(cmdline)
@@ -2211,7 +2211,7 @@ class TestCpWithCRTClient(BaseCRTTransferClientTest):
     def test_respects_no_sign_request_parameter(self):
         filename = self.files.create_file('myfile', 'mycontent')
         cmdline = [
-            's3', 'cp', filename, 's3://bucket/key', '--no-sign-request'
+            's3', '--debug', 'cp', filename, 's3://bucket/key', '--no-sign-request'
         ]
         self.run_command(cmdline)
         self.assert_crt_client_has_no_credential_provider()
@@ -2230,7 +2230,7 @@ class TestCpWithCRTClient(BaseCRTTransferClientTest):
         fake_ca_contents = b"fake ca content"
         ca_bundle = self.files.create_file('fake_ca', fake_ca_contents, mode='wb')
         cmdline = [
-            's3', 'cp', filename, 's3://bucket/key', '--ca-bundle', ca_bundle
+            's3', '--debug', 'cp', filename, 's3://bucket/key', '--ca-bundle', ca_bundle
         ]
         self.run_command(cmdline)
         crt_requests = self.get_crt_make_request_calls()
@@ -2243,7 +2243,7 @@ class TestCpWithCRTClient(BaseCRTTransferClientTest):
         filename = self.files.create_file('myfile', 'mycontent')
         ca_bundle = self.files.create_file('fake_ca', 'mycontent')
         cmdline = [
-            's3', 'cp', filename, 's3://bucket/key', '--ca-bundle', ca_bundle, '--no-verify-ssl'
+            's3', '--debug', 'cp', filename, 's3://bucket/key', '--ca-bundle', ca_bundle, '--no-verify-ssl'
         ]
         self.run_command(cmdline)
         crt_requests = self.get_crt_make_request_calls()
diff --git a/tests/functional/s3/test_mv_command.py b/tests/functional/s3/test_mv_command.py
index 4170bad36..12adda406 100644
--- a/tests/functional/s3/test_mv_command.py
+++ b/tests/functional/s3/test_mv_command.py
@@ -249,7 +249,7 @@ class TestMvWithCRTClient(BaseCRTTransferClientTest):
     def test_upload_move_using_crt_client(self):
         filename = self.files.create_file('myfile', 'mycontent')
         cmdline = [
-            's3', 'mv', filename, 's3://bucket/key',
+            's3', '--debug', 'mv', filename, 's3://bucket/key',
         ]
         self.run_command(cmdline)
         crt_requests = self.get_crt_make_request_calls()
@@ -266,7 +266,7 @@ class TestMvWithCRTClient(BaseCRTTransferClientTest):
     def test_download_move_using_crt_client(self):
         filename = os.path.join(self.files.rootdir, 'myfile')
         cmdline = [
-            's3', 'mv', 's3://bucket/key', filename
+            's3', '--debug', 'mv', 's3://bucket/key', filename
         ]
         self.add_botocore_head_object_response()
         self.add_botocore_delete_object_response()
@@ -285,7 +285,7 @@ class TestMvWithCRTClient(BaseCRTTransferClientTest):

     def test_does_not_use_crt_client_for_copy_moves(self):
         cmdline = [
-            's3', 'mv', 's3://bucket/key', 's3://otherbucket/'
+            's3', '--debug', 'mv', 's3://bucket/key', 's3://otherbucket/'
         ]
         self.add_botocore_head_object_response()
         self.add_botocore_copy_object_response()
diff --git a/tests/functional/s3/test_rm_command.py b/tests/functional/s3/test_rm_command.py
index 3af1d6c48..02b774630 100644
--- a/tests/functional/s3/test_rm_command.py
+++ b/tests/functional/s3/test_rm_command.py
@@ -75,7 +75,7 @@ class TestRmCommand(BaseS3TransferCommandTest):
 class TestRmWithCRTClient(BaseCRTTransferClientTest):
     def test_delete_using_crt_client(self):
         cmdline = [
-            's3', 'rm', 's3://bucket/key'
+            's3', '--debug', 'rm', 's3://bucket/key'
         ]
         self.run_command(cmdline)
         crt_requests = self.get_crt_make_request_calls()
@@ -90,7 +90,7 @@ class TestRmWithCRTClient(BaseCRTTransferClientTest):

     def test_recursive_delete_using_crt_client(self):
         cmdline = [
-            's3', 'rm', 's3://bucket/', '--recursive'
+            's3', '--debug', 'rm', 's3://bucket/', '--recursive'
         ]
         self.add_botocore_list_objects_response(['key1', 'key2'])
         self.run_command(cmdline)
diff --git a/tests/functional/s3/test_sync_command.py b/tests/functional/s3/test_sync_command.py
index 6d3cf6625..7f1782870 100644
--- a/tests/functional/s3/test_sync_command.py
+++ b/tests/functional/s3/test_sync_command.py
@@ -385,7 +385,7 @@ class TestSyncWithCRTClient(BaseCRTTransferClientTest):
     def test_upload_sync_using_crt_client(self):
         filename = self.files.create_file('myfile', 'mycontent')
         cmdline = [
-            's3', 'sync', self.files.rootdir, 's3://bucket/',
+            's3', '--debug', 'sync', self.files.rootdir, 's3://bucket/',
         ]
         self.add_botocore_list_objects_response([])
         self.run_command(cmdline)
@@ -401,7 +401,7 @@ class TestSyncWithCRTClient(BaseCRTTransferClientTest):

     def test_download_sync_using_crt_client(self):
         cmdline = [
-            's3', 'sync', 's3://bucket/', self.files.rootdir,
+            's3', '--debug', 'sync', 's3://bucket/', self.files.rootdir,
         ]
         self.add_botocore_list_objects_response(['key'])
         self.run_command(cmdline)
@@ -418,7 +418,7 @@ class TestSyncWithCRTClient(BaseCRTTransferClientTest):
     def test_upload_sync_with_delete_using_crt_client(self):
         filename = self.files.create_file('a-file', 'mycontent')
         cmdline = [
-            's3', 'sync', self.files.rootdir, 's3://bucket/', '--delete'
+            's3', '--debug', 'sync', self.files.rootdir, 's3://bucket/', '--delete'
         ]
         self.add_botocore_list_objects_response(['delete-this'])
         self.run_command(cmdline)
@@ -442,7 +442,7 @@ class TestSyncWithCRTClient(BaseCRTTransferClientTest):
     def test_download_sync_with_delete_using_crt_client(self):
         self.files.create_file('delete-this', 'content')
         cmdline = [
-            's3', 'sync', 's3://bucket/', self.files.rootdir, '--delete'
+            's3', '--debug', 'sync', 's3://bucket/', self.files.rootdir, '--delete'
         ]
         self.add_botocore_list_objects_response(['key'])
         self.run_command(cmdline)
@@ -459,7 +459,7 @@ class TestSyncWithCRTClient(BaseCRTTransferClientTest):

     def test_does_not_use_crt_client_for_copy_syncs(self):
         cmdline = [
-            's3', 'sync', 's3://bucket/', 's3://otherbucket/'
+            's3', '--debug', 'sync', 's3://bucket/', 's3://otherbucket/'
         ]
         self.add_botocore_list_objects_response(['key'])
         self.add_botocore_list_objects_response([])

And the following debugging patch for aws-c-io submodule in python-awscrt:

diff --git a/source/linux/epoll_event_loop.c b/source/linux/epoll_event_loop.c
index 094a7836..86e4a8a9 100644
--- a/source/linux/epoll_event_loop.c
+++ b/source/linux/epoll_event_loop.c
@@ -149,7 +149,8 @@ struct aws_event_loop *aws_event_loop_new_default_with_options(

     epoll_loop->epoll_fd = epoll_create(100);
     if (epoll_loop->epoll_fd < 0) {
-        AWS_LOGF_FATAL(AWS_LS_IO_EVENT_LOOP, "id=%p: Failed to open epoll handle.", (void *)loop);
+        int error_code = errno;
+        AWS_LOGF_FATAL(AWS_LS_IO_EVENT_LOOP, "id=%p: Failed to open epoll handle. error=%d", (void *)loop, error_code);
         aws_raise_error(AWS_ERROR_SYS_CALL_FAILURE);
         goto clean_up_epoll;
     }

I can see this in test logs:

[FATAL] [2024-04-02T16:41:36Z] [000070a056e2b740] [event-loop] - id=0x62604dc4af80: Failed to open epoll handle. error=24

I guess the error 24 (EMFILE, Too many open files) is caused by too high
parallelism, so I raise the ulimit as a workaround. It seems the error
never happens again with the new ulimit.

1 files changed, 3 insertions, 0 deletions

diff --git a/PKGBUILD b/PKGBUILD
index 099f89f63153..05dc71ee453c 100644
--- a/PKGBUILD
+++ b/PKGBUILD
@@ -103,6 +103,9 @@ build() {
 }
 
 check() {
+  # Avoid intermittent test failures, see git commit messages
+  ulimit -S -n 4096
+
   cd awscli-$pkgver
 
   export AWS_SECRET_ACCESS_KEY=fake_key