new version

4 files changed, 7 insertions, 470 deletions

diff --git a/.SRCINFO b/.SRCINFO
index 876490ef3206..a07d21329593 100644
--- a/.SRCINFO
+++ b/.SRCINFO
@@ -1,6 +1,6 @@
 pkgbase = libs3-git
 	pkgdesc = C Library and Tools for Amazon S3 Access
-	pkgver = 258.fd8b149
+	pkgver = 267.537bb7b
 	pkgrel = 2
 	url = https://github.com/bji/libs3
 	arch = i686
@@ -13,14 +13,10 @@ pkgbase = libs3-git
 	depends = libxml2
 	depends = openssl
 	depends = curl
-	source = git://github.com/bji/libs3.git#commit=fd8b149044e429ad30dc4c918f0713cdd40aadd2
+	source = git://github.com/bji/libs3.git#commit=537bb7b59bbf3d9eb322eb807d329bb701075a88
 	source = added_s3_versioning_support.patch
-	source = extension_to_support_curl_multi_socket_action_operation.patch
-	source = fixes_segfault_while_putting_large_files_read_from_stdin.patch
 	sha256sums = SKIP
 	sha256sums = 0549154da30ef026b334a83cb69e168f7bb243a17569f54bbc57f0840d76f8c2
-	sha256sums = 1605ff732130f5644d17d70bffe546297214d287c6eeee1db885bd302b8168c0
-	sha256sums = 8e1b9ebbf98c1db023bb0a735d6dfba224c99e2753c28bb0df1444b530641a62
 
 pkgname = libs3-git
 
diff --git a/PKGBUILD b/PKGBUILD
index c8954be5eb65..f0c9b79ae90f 100644
--- a/PKGBUILD
+++ b/PKGBUILD
@@ -2,7 +2,7 @@
 # Contributor: Bryan Ischo <bryan@ischo.com>
 pkgname=libs3-git
 _gitname=libs3
-pkgver=258.fd8b149
+pkgver=267.537bb7b
 pkgrel=2
 pkgdesc="C Library and Tools for Amazon S3 Access"
 arch=('i686' 'x86_64')
@@ -17,15 +17,11 @@ replaces=()
 backup=()
 options=()
 install=
-source=("git://github.com/bji/${_gitname}.git#commit=fd8b149044e429ad30dc4c918f0713cdd40aadd2"
-        'added_s3_versioning_support.patch'
-        'extension_to_support_curl_multi_socket_action_operation.patch'
-        'fixes_segfault_while_putting_large_files_read_from_stdin.patch')
+source=("git://github.com/bji/${_gitname}.git#commit=537bb7b59bbf3d9eb322eb807d329bb701075a88"
+        'added_s3_versioning_support.patch')
 noextract=()
 sha256sums=('SKIP'
-         '0549154da30ef026b334a83cb69e168f7bb243a17569f54bbc57f0840d76f8c2'
-         '1605ff732130f5644d17d70bffe546297214d287c6eeee1db885bd302b8168c0'
-         '8e1b9ebbf98c1db023bb0a735d6dfba224c99e2753c28bb0df1444b530641a62')
+            '0549154da30ef026b334a83cb69e168f7bb243a17569f54bbc57f0840d76f8c2')
 
 
 
@@ -33,10 +29,6 @@ prepare() {
  cd "$_gitname"
  # https://github.com/bji/libs3/pull/70
  patch -Np1 -i "${srcdir}/added_s3_versioning_support.patch"
- # https://github.com/bji/libs3/pull/69
- patch -Np1 -i "${srcdir}/extension_to_support_curl_multi_socket_action_operation.patch"
- # https://github.com/bji/libs3/pull/68/commits/05deb102556cd8a06da0940f90f7d22e4f61f162
- patch -Np1 -i "${srcdir}/fixes_segfault_while_putting_large_files_read_from_stdin.patch"
 }
 
 pkgver() {
@@ -46,5 +38,5 @@ pkgver() {
 
 package() {
   cd "$_gitname"
-  CFLAGS=-Wno-format-truncation DESTDIR=$pkgdir/usr make install || return 1
+  CFLAGS="-Wno-stringop-overflow -Wno-format-truncation" DESTDIR=$pkgdir/usr make install || return 1
 }
diff --git a/extension_to_support_curl_multi_socket_action_operation.patch b/extension_to_support_curl_multi_socket_action_operation.patch
deleted file mode 100644
index cba1add53c30..000000000000
--- a/extension_to_support_curl_multi_socket_action_operation.patch
+++ /dev/null
@@ -1,393 +0,0 @@
-From e6ebeb70fc4263ab66caba4e00bdff9da278c5f0 Mon Sep 17 00:00:00 2001
-From: Alexei Potashnik <alexeip@gmail.com>
-Date: Wed, 18 Oct 2017 10:10:59 -0700
-Subject: [PATCH] extension to support curl_multi_socket_action operation
-
-Make libs3 work with curl_multi_socket_action mode where libs3 user
-will drive socket polling instructed directly by curl through a set
-of callbacks.
-
-This is an addition to the curl_multi_perform mode where libs3 does
-the polling itself.
----
- inc/libs3.h           |  87 +++++++++++++++++++++++++++++++
- inc/request_context.h |  12 +++++
- src/request.c         |  12 ++++-
- src/request_context.c | 139 +++++++++++++++++++++++++++++++++-----------------
- 4 files changed, 202 insertions(+), 48 deletions(-)
-
-diff --git a/inc/libs3.h b/inc/libs3.h
-index aab820e..47cc19b 100644
---- a/inc/libs3.h
-+++ b/inc/libs3.h
-@@ -1240,6 +1240,28 @@ typedef S3Status (S3MultipartCommitResponseCallback)(const char *location,
-                                                      void *callbackData);
- 
- 
-+/**
-+ * Mechanism for S3 application to customize each CURL easy request
-+ * associated with the given S3 request context.
-+ *
-+ * This callback can be optinally configured using S3_create_request_context_ex
-+ * and will be invoked every time a new CURL request is created in the
-+ * context of the given CURLM handle. Invocation will occur after
-+ * libs3 has finished configuring its own options of CURL, but before
-+ * CURL is started.
-+ *
-+ * @param curl_multi is the CURLM handle associated with this context.
-+ * @param curl_easy is the CURL request being created.
-+ * @param setupData is the setupCurlCallbackData parameter passed to
-+ *        S3_create_request_context_ex.
-+ * @return S3StatusOK to continue processing the request, anything else to
-+ *         immediately abort the request and pass this status
-+ *         to the S3ResponseCompleteCallback for this request.
-+ **/
-+typedef S3Status (*S3SetupCurlCallback)(void *curlMulti, void *curlEasy,
-+                                        void *setupData);
-+
-+
- /** **************************************************************************
-  * Callback Structures
-  ************************************************************************** **/
-@@ -1583,6 +1605,51 @@ int S3_status_is_retryable(S3Status status);
- S3Status S3_create_request_context(S3RequestContext **requestContextReturn);
- 
- 
-+/**
-+ * Extended version of S3_create_request_context used to create S3RequestContext
-+ * for curl_multi_socket_action CURLM handles that will be managed by libs3 user.
-+ * This type of handles offer better performance for applications with large
-+ * number of simultaneous connections. For details, see MULTI_SOCKET chapter here:
-+ * https://curl.haxx.se/libcurl/c/libcurl-multi.html
-+ *
-+ * In this mode libs3 user will
-+ *  - create its own CURLM using curl_multi_init()
-+ *  - configure it for its own handlers using
-+ *    CURLMOPT_SOCKETFUNCTION/CURLMOPT_TIMERFUNCTION/etc
-+ *  - use S3_create_request_context_ex to create S3RequestContext
-+ *    for the above CURLM handle
-+ *  - start S3 request
-+ *  - every time setupCurlCallback is called, will configure new CURL
-+ *    object with its own handlers using
-+ *    CURLOPT_OPENSOCKETFUNCTION/CURLOPT_CLOSESOCKETFUNCTION/etc
-+ *  - the moment libs3 adds CURL object to CURLM handle, curl will start
-+ *    communicating directly with libs3 user to drive socket operations,
-+ *    where libs3 user will be responsible for calling curl_multi_socket_action
-+ *    when necessary.
-+ *  - whenever curl_multi_socket_action indicates change in running_handles
-+ *    libs3 user should call S3_process_request_context to let libs3 process
-+ *    any completed curl transfers and notify back to libs3 user if that was
-+ *    the final transfer for a given S3 request.
-+ *
-+ * @param requestContextReturn returns the newly-created S3RequestContext
-+ *        structure, which if successfully returned, must be destroyed via a
-+ *        call to S3_destroy_request_context when it is no longer needed.  If
-+ *        an error status is returned from this function, then
-+ *        requestContextReturn will not have been filled in, and
-+ *        S3_destroy_request_context should not be called on it
-+ * @param curlMulti is the CURLM handle to be associated with this context.
-+ * @param setupCurlCallback is an optional callback routine to be invoked
-+ *        by libs3 every time another CURL request is being created for
-+ *        use in this context.
-+ * @param setupCurlCallbackData is an opaque data to be passed to
-+ *        setupCurlCallback.
-+ **/
-+S3Status S3_create_request_context_ex(S3RequestContext **requestContextReturn,
-+                                      void *curlMulti,
-+                                      S3SetupCurlCallback setupCurlCallback,
-+                                      void *setupCurlCallbackData);
-+
-+
- /**
-  * Destroys an S3RequestContext which was created with
-  * S3_create_request_context.  Any requests which are currently being
-@@ -1640,6 +1707,26 @@ S3Status S3_runonce_request_context(S3RequestContext *requestContext,
-                                     int *requestsRemainingReturn);
- 
- 
-+/**
-+ * Extract and finish requests completed by curl multi handle mechanism
-+ * in curl_multi_socket_action mode. Should be called by libs3 user when
-+ * curl_multi_socket_action indicates a change in running_handles.
-+ *
-+ * @param requestContext is the S3RequestContext to process
-+ * @return One of:
-+ *         S3StatusOK if request processing proceeded without error
-+ *         S3StatusConnectionFailed if the socket connection to the server
-+ *             failed
-+ *         S3StatusServerFailedVerification if the SSL certificate of the
-+ *             server could not be verified.
-+ *         S3StatusInternalError if an internal error prevented the
-+ *             S3RequestContext from running one or more requests
-+ *         S3StatusOutOfMemory if requests could not be processed due to
-+ *             an out of memory error
-+ **/
-+S3Status S3_process_request_context(S3RequestContext *requestContext);
-+
-+
- /**
-  * This function, in conjunction allows callers to manually manage a set of
-  * requests using an S3RequestContext.  This function returns the set of file
-diff --git a/inc/request_context.h b/inc/request_context.h
-index 229ef35..b697cc0 100644
---- a/inc/request_context.h
-+++ b/inc/request_context.h
-@@ -29,14 +29,26 @@
- 
- #include "libs3.h"
- 
-+
-+typedef enum
-+{
-+    S3CurlModeMultiPerform                                  ,
-+    S3CurlModeMultiSocket                                   ,
-+} S3CurlMode;
-+
-+
- struct S3RequestContext
- {
-     CURLM *curlm;
-+    S3CurlMode curl_mode;
-     
-     int verifyPeerSet;
-     long verifyPeer;
- 
-     struct Request *requests;
-+
-+    S3SetupCurlCallback setupCurlCallback;
-+    void *setupCurlCallbackData;
- };
- 
- 
-diff --git a/src/request.c b/src/request.c
-index 63174d0..156ea32 100644
---- a/src/request.c
-+++ b/src/request.c
-@@ -1301,6 +1301,7 @@ static void request_deinitialize(Request *request)
- 
- static S3Status request_get(const RequestParams *params,
-                             const RequestComputedValues *values,
-+                            const S3RequestContext *context,
-                             Request **reqReturn)
- {
-     Request *request = 0;
-@@ -1360,6 +1361,15 @@ static S3Status request_get(const RequestParams *params,
-         return status;
-     }
- 
-+    if (context && context->setupCurlCallback &&
-+        (status = context->setupCurlCallback(
-+                context->curlm, request->curl,
-+                context->setupCurlCallbackData)) != S3StatusOK) {
-+        curl_easy_cleanup(request->curl);
-+        free(request);
-+        return status;
-+    }
-+
-     request->propertiesCallback = params->propertiesCallback;
- 
-     request->toS3Callback = params->toS3Callback;
-@@ -1548,7 +1558,7 @@ void request_perform(const RequestParams *params, S3RequestContext *context)
-     }
- 
-     // Get an initialized Request structure now
--    if ((status = request_get(params, &computed, &request)) != S3StatusOK) {
-+    if ((status = request_get(params, &computed, context, &request)) != S3StatusOK) {
-         return_status(status);
-     }
-     if (context && context->verifyPeerSet) {
-diff --git a/src/request_context.c b/src/request_context.c
-index cf82f2d..eeecbd6 100644
---- a/src/request_context.c
-+++ b/src/request_context.c
-@@ -31,7 +31,10 @@
- #include "request_context.h"
- 
- 
--S3Status S3_create_request_context(S3RequestContext **requestContextReturn)
-+S3Status S3_create_request_context_ex(S3RequestContext **requestContextReturn,
-+                                      CURLM *curlm,
-+                                      S3SetupCurlCallback setupCurlCallback,
-+                                      void *setupCurlCallbackData)
- {
-     *requestContextReturn = 
-         (S3RequestContext *) malloc(sizeof(S3RequestContext));
-@@ -40,19 +43,35 @@ S3Status S3_create_request_context(S3RequestContext **requestContextReturn)
-         return S3StatusOutOfMemory;
-     }
-     
--    if (!((*requestContextReturn)->curlm = curl_multi_init())) {
--        free(*requestContextReturn);
--        return S3StatusOutOfMemory;
-+    if (curlm) {
-+        (*requestContextReturn)->curlm = curlm;
-+        (*requestContextReturn)->curl_mode = S3CurlModeMultiSocket;
-+    }
-+    else {
-+        if (!((*requestContextReturn)->curlm = curl_multi_init())) {
-+            free(*requestContextReturn);
-+            return S3StatusOutOfMemory;
-+        }
-+
-+        (*requestContextReturn)->curl_mode = S3CurlModeMultiPerform;
-     }
- 
-     (*requestContextReturn)->requests = 0;
-     (*requestContextReturn)->verifyPeer = 0;
-     (*requestContextReturn)->verifyPeerSet = 0;
-+    (*requestContextReturn)->setupCurlCallback = setupCurlCallback;
-+    (*requestContextReturn)->setupCurlCallbackData = setupCurlCallbackData;
- 
-     return S3StatusOK;
- }
- 
- 
-+S3Status S3_create_request_context(S3RequestContext **requestContextReturn)
-+{
-+    return S3_create_request_context_ex(requestContextReturn, NULL, NULL, NULL);
-+}
-+
-+
- void S3_destroy_request_context(S3RequestContext *requestContext)
- {
-     // For each request in the context, remove curl handle, call back its done
-@@ -68,7 +87,8 @@ void S3_destroy_request_context(S3RequestContext *requestContext)
-         r = rNext;
-     } while (r != rFirst);
- 
--    curl_multi_cleanup(requestContext->curlm);
-+    if (requestContext->curl_mode == S3CurlModeMultiPerform)
-+        curl_multi_cleanup(requestContext->curlm);
- 
-     free(requestContext);
- }
-@@ -109,10 +129,62 @@ S3Status S3_runall_request_context(S3RequestContext *requestContext)
- }
- 
- 
-+static S3Status process_request_context(S3RequestContext *requestContext, int *retry)
-+{
-+    CURLMsg *msg;
-+    int junk;
-+
-+    *retry = 0;
-+
-+    while ((msg = curl_multi_info_read(requestContext->curlm, &junk))) {
-+        if (msg->msg != CURLMSG_DONE) {
-+            return S3StatusInternalError;
-+        }
-+        Request *request;
-+        if (curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE,
-+                              (char **) (char *) &request) != CURLE_OK) {
-+            return S3StatusInternalError;
-+        }
-+        // Remove the request from the list of requests
-+        if (request->prev == request->next) {
-+            // It was the only one on the list
-+            requestContext->requests = 0;
-+        }
-+        else {
-+            // It doesn't matter what the order of them are, so just in
-+            // case request was at the head of the list, put the one after
-+            // request to the head of the list
-+            requestContext->requests = request->next;
-+            request->prev->next = request->next;
-+            request->next->prev = request->prev;
-+        }
-+        if ((msg->data.result != CURLE_OK) &&
-+            (request->status == S3StatusOK)) {
-+            request->status = request_curl_code_to_status(
-+                msg->data.result);
-+        }
-+        if (curl_multi_remove_handle(requestContext->curlm,
-+                                     msg->easy_handle) != CURLM_OK) {
-+            return S3StatusInternalError;
-+        }
-+        // Finish the request, ensuring that all callbacks have been made,
-+        // and also releases the request
-+        request_finish(request);
-+        // Now, since a callback was made, there may be new requests
-+        // queued up to be performed immediately, so do so
-+        *retry = 1;
-+    }
-+
-+    return S3StatusOK;
-+}
-+
-+
- S3Status S3_runonce_request_context(S3RequestContext *requestContext, 
-                                     int *requestsRemainingReturn)
- {
-+    S3Status s3_status;
-     CURLMcode status;
-+    int retry;
- 
-     do {
-         status = curl_multi_perform(requestContext->curlm,
-@@ -128,51 +200,24 @@ S3Status S3_runonce_request_context(S3RequestContext *requestContext,
-             return S3StatusInternalError;
-         }
- 
--        CURLMsg *msg;
--        int junk;
--        while ((msg = curl_multi_info_read(requestContext->curlm, &junk))) {
--            if (msg->msg != CURLMSG_DONE) {
--                return S3StatusInternalError;
--            }
--            Request *request;
--            if (curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, 
--                                  (char **) (char *) &request) != CURLE_OK) {
--                return S3StatusInternalError;
--            }
--            // Remove the request from the list of requests
--            if (request->prev == request->next) {
--                // It was the only one on the list
--                requestContext->requests = 0;
--            }
--            else {
--                // It doesn't matter what the order of them are, so just in
--                // case request was at the head of the list, put the one after
--                // request to the head of the list
--                requestContext->requests = request->next;
--                request->prev->next = request->next;
--                request->next->prev = request->prev;
--            }
--            if ((msg->data.result != CURLE_OK) &&
--                (request->status == S3StatusOK)) {
--                request->status = request_curl_code_to_status
--                    (msg->data.result);
--            }
--            if (curl_multi_remove_handle(requestContext->curlm, 
--                                         msg->easy_handle) != CURLM_OK) {
--                return S3StatusInternalError;
--            }
--            // Finish the request, ensuring that all callbacks have been made,
--            // and also releases the request
--            request_finish(request);
--            // Now, since a callback was made, there may be new requests 
--            // queued up to be performed immediately, so do so
--            status = CURLM_CALL_MULTI_PERFORM;
--        }
--    } while (status == CURLM_CALL_MULTI_PERFORM);
-+        s3_status = process_request_context(requestContext, &retry);
-+    } while (s3_status == S3StatusOK &&
-+             (status == CURLM_CALL_MULTI_PERFORM || retry));
- 
--    return S3StatusOK;
-+    return s3_status;
- }
- 
-+
-+S3Status S3_process_request_context(S3RequestContext *requestContext)
-+{
-+    int retry;
-+    /* In curl_multi_socket_action mode any new requests created during
-+       the following call will have already started associated socket
-+       operations, so no need to retry here */
-+    return process_request_context(requestContext, &retry);
-+}
-+
-+
- S3Status S3_get_request_context_fdsets(S3RequestContext *requestContext,
-                                        fd_set *readFdSet, fd_set *writeFdSet,
-                                        fd_set *exceptFdSet, int *maxFd)
diff --git a/fixes_segfault_while_putting_large_files_read_from_stdin.patch b/fixes_segfault_while_putting_large_files_read_from_stdin.patch
deleted file mode 100644
index a177a97f7c7c..000000000000
--- a/fixes_segfault_while_putting_large_files_read_from_stdin.patch
+++ /dev/null
@@ -1,58 +0,0 @@
-From 05deb102556cd8a06da0940f90f7d22e4f61f162 Mon Sep 17 00:00:00 2001
-From: "sergey.dobrodey" <sergey.dobrodey@hotmail.com>
-Date: Mon, 2 Oct 2017 13:13:32 +0300
-Subject: [PATCH] fixes https://github.com/bji/libs3/issues/21
-
----
- src/response_headers_handler.c | 3 ++-
- src/s3.c                       | 7 +++++--
- 2 files changed, 7 insertions(+), 3 deletions(-)
-
-diff --git a/src/response_headers_handler.c b/src/response_headers_handler.c
-index 70046bc..bc4c175 100644
---- a/src/response_headers_handler.c
-+++ b/src/response_headers_handler.c
-@@ -145,7 +145,8 @@ void response_headers_handler_add(ResponseHeadersHandler *handler,
-         string_multibuffer_add(handler->responsePropertyStrings, c, 
-                                valuelen, fit);
-     }
--    else if (!strncasecmp(header, "ETag", namelen)) {
-+    else if ((!strncasecmp(header, "ETag", namelen)) 
-+         || (!strncasecmp(header, "Etag", namelen))) { // some servers reply with Etag header
-         responseProperties->eTag = 
-             string_multibuffer_current(handler->responsePropertyStrings);
-         string_multibuffer_add(handler->responsePropertyStrings, c, 
-diff --git a/src/s3.c b/src/s3.c
-index b8d4405..88d3ddb 100644
---- a/src/s3.c
-+++ b/src/s3.c
-@@ -489,6 +489,7 @@ static void growbuffer_read(growbuffer **gb, int amt, int *amtReturn,
-             buf->next->prev = buf->prev;
-         }
-         free(buf);
-+        buf = NULL;
-     }
- }
- 
-@@ -2436,6 +2437,7 @@ static void put_object(int argc, char **argv, int optindex,
-                         MULTIPART_CHUNK_SIZE);
- 
-         MultipartPartData partData;
-+        memset(&partData, 0, sizeof(MultipartPartData));
-         int partContentLength = 0;
- 
-         S3MultipartInitialHandler handler = {
-@@ -2486,10 +2488,11 @@ static void put_object(int argc, char **argv, int optindex,
- upload:
-         todoContentLength -= MULTIPART_CHUNK_SIZE * manager.next_etags_pos;
-         for (seq = manager.next_etags_pos + 1; seq <= totalSeq; seq++) {
--            memset(&partData, 0, sizeof(MultipartPartData));
-             partData.manager = &manager;
-             partData.seq = seq;
--            partData.put_object_data = data;
-+            if (partData.put_object_data.gb==NULL) {
-+              partData.put_object_data = data;
-+            }
-             partContentLength = ((contentLength > MULTIPART_CHUNK_SIZE) ?
-                                  MULTIPART_CHUNK_SIZE : contentLength);
-             printf("%s Part Seq %d, length=%d\n", srcSize ? "Copying" : "Sending", seq, partContentLength);