Switch back from Soup to cURL for HTTP(S)

It turns out that people do prefer cURL over libsoup for
minimal/embedded installations, and for some odd reason *do* want
libappstream on those...
Since our use of libsoup is quite light and libappstream is supposed to
not have a lot of heavy dependencies, we just switch back to cURL as
used before. This time though, it's wrapped in a new internal AsCurl
class for convenience.

README.md

@@ -49,6 +49,7 @@ you may want to take a look at the [AppStream Generator](https://github.com/ximi
  * GObject-Introspection
  * libxml2
  * libyaml
+ * libcurl (>= 7.62)
  * LMDB
 
 #### Optional

meson.build

@@ -90,7 +90,7 @@ glib_dep = dependency('glib-2.0', version: '>=2.58')
 gobject_dep = dependency('gobject-2.0', version: '>=2.58')
 gio_dep = dependency('gio-2.0', version: '>=2.58')
 gio_unix_dep = dependency('gio-unix-2.0', version: '>=2.58')
-soup_dep = dependency('libsoup-2.4', version: '>= 2.56')
+curl_dep = dependency('libcurl', version : '>= 7.62')
 xml2_dep = dependency('libxml-2.0')
 yaml_dep = dependency('yaml-0.1')
 lmdb_dep = dependency('lmdb', required: false)

po/POTFILES.in

@@ -11,6 +11,7 @@ src/as-category.c
 src/as-checksum.c
 src/as-component.c
 src/as-content-rating.c
+src/as-curl.c
 src/as-desktop-entry.c
 src/as-distro-details.c
 src/as-distro-extras.c

src/as-curl.c

+/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*-
+ *
+ * Copyright (C) 2020-2021 Matthias Klumpp <matthias@tenstral.net>
+ *
+ * Licensed under the GNU Lesser General Public License Version 2.1
+ *
+ * This library is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 2.1 of the license, or
+ * (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this library.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/**
+ * SECTION:as-curl
+ * @short_description: Internal convenience wrapper around some cURL functions for AppStream
+ * @include: appstream.h
+ */
+
+#include "config.h"
+#include "as-curl.h"
+
+#include <curl/curl.h>
+
+struct _AsCurl
+{
+	GObject parent_instance;
+};
+
+typedef struct
+{
+	CURL		*curl;
+	const gchar	*user_agent;
+} AsCurlPrivate;
+
+G_DEFINE_TYPE_WITH_PRIVATE (AsCurl, as_curl, G_TYPE_OBJECT)
+#define GET_PRIVATE(o) (as_curl_get_instance_private (o))
+
+G_DEFINE_AUTOPTR_CLEANUP_FUNC(CURLU, curl_url_cleanup)
+
+G_DEFINE_QUARK (AsCurlError, as_curl_error)
+
+/**
+ * as_curl_is_url:
+ *
+ * Check if the URL is valid.
+ */
+gboolean
+as_curl_is_url (const gchar *url)
+{
+	g_autoptr(CURLU) cu = curl_url ();
+	return curl_url_set (cu, CURLUPART_URL, url, 0) == CURLUE_OK;
+}
+
+static void
+as_curl_finalize (GObject *object)
+{
+	AsCurl *acurl = AS_CURL (object);
+	AsCurlPrivate *priv = GET_PRIVATE (acurl);
+
+	if (priv->curl != NULL)
+		curl_easy_cleanup (priv->curl);
+
+	G_OBJECT_CLASS (as_curl_parent_class)->finalize (object);
+}
+
+static void
+as_curl_init (AsCurl *acurl)
+{
+	AsCurlPrivate *priv = GET_PRIVATE (acurl);
+
+	priv->user_agent = "appstream/" PACKAGE_VERSION;
+}
+
+static void
+as_curl_class_init (AsCurlClass *klass)
+{
+	GObjectClass *object_class = G_OBJECT_CLASS (klass);
+	object_class->finalize = as_curl_finalize;
+}
+
+static size_t
+as_curl_download_write_bytearray_cb (char *ptr, size_t size, size_t nmemb, void *udata)
+{
+	GByteArray *buf = (GByteArray *) udata;
+	gsize realsize = size * nmemb;
+	g_byte_array_append (buf, (const guint8 *) ptr, realsize);
+	return realsize;
+}
+
+/**
+ * as_curl_download_bytes:
+ *
+ * Download an URL as GBytes.
+ */
+GBytes*
+as_curl_download_bytes (AsCurl *acurl, const gchar *url, GError **error)
+{
+	AsCurlPrivate *priv = GET_PRIVATE (acurl);
+	CURLcode res;
+	gchar errbuf[CURL_ERROR_SIZE] = { '\0' };
+	glong status_code = 0;
+	g_autoptr(GByteArray) buf = g_byte_array_new ();
+
+	curl_easy_setopt (priv->curl, CURLOPT_URL, url);
+	curl_easy_setopt (priv->curl, CURLOPT_ERRORBUFFER, errbuf);
+	curl_easy_setopt (priv->curl, CURLOPT_WRITEFUNCTION, as_curl_download_write_bytearray_cb);
+	curl_easy_setopt (priv->curl, CURLOPT_WRITEDATA, buf);
+
+	res = curl_easy_perform (priv->curl);
+	curl_easy_getinfo (priv->curl, CURLINFO_RESPONSE_CODE, &status_code);
+	if (res != CURLE_OK) {
+		g_debug ("cURL status-code was %ld", status_code);
+		if (status_code == 429) {
+			g_set_error (error,
+				     AS_CURL_ERROR,
+				     AS_CURL_ERROR_REMOTE,
+				     "Failed to download due to server limit");
+			return NULL;
+		}
+		if (errbuf[0] != '\0') {
+			g_set_error (error,
+				     AS_CURL_ERROR,
+				     AS_CURL_ERROR_DOWNLOAD,
+				     "Failed to download file: %s",
+				     errbuf);
+			return NULL;
+		}
+		g_set_error (error,
+			     AS_CURL_ERROR,
+			     AS_CURL_ERROR_DOWNLOAD,
+			     "Failed to download file: %s",
+			     curl_easy_strerror (res));
+		return NULL;
+	}
+	if (status_code == 404) {
+		g_set_error (error,
+			     AS_CURL_ERROR,
+			     AS_CURL_ERROR_REMOTE,
+			     "URL was not found on server.");
+		return NULL;
+	} else if (status_code != 200) {
+		g_set_error (error,
+			     AS_CURL_ERROR,
+			     AS_CURL_ERROR_REMOTE,
+			     "Unexpected status code: %ld",
+			     status_code);
+		return NULL;
+	}
+
+	return g_byte_array_free_to_bytes (g_steal_pointer (&buf));
+}
+
+/**
+ * as_curl_new:
+ *
+ * Creates a new #AsCurl.
+ **/
+AsCurl*
+as_curl_new (GError **error)
+{
+	AsCurlPrivate *priv;
+	const gchar *http_proxy;
+	g_autoptr(AsCurl) acurl = g_object_new (AS_TYPE_CURL, NULL);
+	priv = GET_PRIVATE (acurl);
+
+	priv->curl = curl_easy_init ();
+	if (priv->curl == NULL) {
+		g_set_error_literal (error,
+				     AS_CURL_ERROR,
+				     AS_CURL_ERROR_FAILED,
+				     "Failed to setup networking, could not initialize cURL.");
+		return NULL;
+	}
+
+	if (g_getenv ("AS_CURL_VERBOSE") != NULL)
+		curl_easy_setopt (priv->curl, CURLOPT_VERBOSE, 1L);
+
+	curl_easy_setopt (priv->curl, CURLOPT_USERAGENT, priv->user_agent);
+	curl_easy_setopt (priv->curl, CURLOPT_CONNECTTIMEOUT, 60L);
+	curl_easy_setopt (priv->curl, CURLOPT_NOPROGRESS, 1L);
+	curl_easy_setopt (priv->curl, CURLOPT_FOLLOWLOCATION, 1L);
+	curl_easy_setopt (priv->curl, CURLOPT_MAXREDIRS, 8L);
+
+	/* read common proxy environment variables */
+	http_proxy = g_getenv ("https_proxy");
+	if (http_proxy == NULL)
+		http_proxy = g_getenv ("HTTPS_PROXY");
+	if (http_proxy == NULL)
+		http_proxy = g_getenv ("http_proxy");
+	if (http_proxy == NULL)
+		http_proxy = g_getenv ("HTTP_PROXY");
+	if (http_proxy != NULL && strlen (http_proxy) > 0)
+		curl_easy_setopt (priv->curl, CURLOPT_PROXY, http_proxy);
+
+	return AS_CURL (g_steal_pointer (&acurl));
+}

src/as-curl.h

+/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*-
+ *
+ * Copyright (C) 2020-2021 Matthias Klumpp <matthias@tenstral.net>
+ *
+ * Licensed under the GNU Lesser General Public License Version 2.1
+ *
+ * This library is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 2.1 of the license, or
+ * (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this library.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#if !defined (AS_COMPILATION)
+#error "Can not use internal AppStream API from external project."
+#endif
+
+#pragma once
+
+#include <glib-object.h>
+
+G_BEGIN_DECLS
+
+#define AS_TYPE_CURL (as_curl_get_type ())
+G_DECLARE_FINAL_TYPE (AsCurl, as_curl, AS, CURL, GObject)
+
+/**
+ * AsCurlError:
+ * @AS_CURL_ERROR_FAILED:	Generic failure.
+ * @AS_CURL_ERROR_REMOTE:	Some issue happened on the remote side.
+ * @AS_CURL_ERROR_DOWNLOAD:	Download failed.
+ *
+ * An cURL error.
+ **/
+typedef enum {
+	AS_CURL_ERROR_FAILED,
+	AS_CURL_ERROR_REMOTE,
+	AS_CURL_ERROR_DOWNLOAD,
+	/*< private >*/
+	AS_CURL_ERROR_LAST
+} AsCurlError;
+
+#define	AS_CURL_ERROR	as_curl_error_quark ()
+GQuark			as_curl_error_quark (void);
+
+AsCurl			*as_curl_new (GError **error);
+
+GBytes			*as_curl_download_bytes (AsCurl *acurl,
+						 const gchar *url,
+						 GError **error);
+
+gboolean		as_curl_is_url (const gchar *url);
+
+G_END_DECLS

src/as-validator.c

@@ -36,15 +36,13 @@
 #include <libxml/parser.h>
 #include <string.h>
 
-#include <libsoup/soup.h>
-#include <libsoup/soup-status.h>
-
 #include "as-validator.h"
 #include "as-validator-issue.h"
 #include "as-validator-issue-tag.h"
 
 #include "as-utils.h"
 #include "as-utils-private.h"
+#include "as-curl.h"
 #include "as-vercmp.h"
 #include "as-spdx.h"
 #include "as-component.h"
@@ -62,7 +60,7 @@ typedef struct
 	gchar		*current_fname;
 
 	gboolean	check_urls;
-	SoupSession	*soup_session;
+	AsCurl		*acurl;
 } AsValidatorPrivate;
 
 G_DEFINE_TYPE_WITH_PRIVATE (AsValidator, as_validator, G_TYPE_OBJECT)
@@ -99,7 +97,6 @@ as_validator_init (AsValidator *validator)
 							g_str_equal,
 							g_free,
 							(GDestroyNotify) g_ptr_array_unref);
-	priv->soup_session = NULL;
 	priv->current_fname = NULL;
 	priv->current_cpt = NULL;
 	priv->check_urls = FALSE;
@@ -123,8 +120,8 @@ as_validator_finalize (GObject *object)
 	if (priv->current_cpt != NULL)
 		g_object_unref (priv->current_cpt);
 
-	if (priv->soup_session != NULL)
-		g_object_unref (priv->soup_session);
+	if (priv->acurl != NULL)
+		g_object_unref (priv->acurl);
 
 	G_OBJECT_CLASS (as_validator_parent_class)->finalize (object);
 }
@@ -298,26 +295,21 @@ static gboolean
 as_validator_setup_networking (AsValidator *validator)
 {
 	AsValidatorPrivate *priv = GET_PRIVATE (validator);
+	g_autoptr(GError) tmp_error = NULL;
 
 	/* check if we are already initialized */
-	if (priv->soup_session != NULL)
+	if (priv->acurl != NULL)
 		return TRUE;
 
 	/* don't initialize if no URLs should be checked */
 	if (!priv->check_urls)
 		return TRUE;
 
-	priv->soup_session = soup_session_new_with_options (SOUP_SESSION_USER_AGENT,
-							    "appstream-validator",
-							    SOUP_SESSION_TIMEOUT,
-							    90,
-							    NULL);
-	if (priv->soup_session == NULL) {
-		g_critical ("Failed to set up networking support");
+	priv->acurl = as_curl_new (&tmp_error);
+	if (priv->acurl == NULL) {
+		g_critical ("Failed to set up networking support: %s", tmp_error->message);
 		return FALSE;
 	}
-	soup_session_add_feature_by_type (priv->soup_session,
-					  SOUP_TYPE_PROXY_RESOLVER_DEFAULT);
 	return TRUE;
 }
 
@@ -331,9 +323,8 @@ static gboolean
 as_validator_validate_web_url (AsValidator *validator, xmlNode *node, const gchar *url, const gchar *tag)
 {
 	AsValidatorPrivate *priv = GET_PRIVATE (validator);
-	guint status_code;
-	g_autoptr(SoupMessage) msg = NULL;
-	g_autoptr(SoupURI) base_uri = NULL;
+	g_autoptr(GBytes) bytes = NULL;
+	g_autoptr(GError) tmp_error = NULL;
 
 	/* we don't check mailto URLs */
 	if (g_str_has_prefix (url, "mailto:"))
@@ -347,43 +338,38 @@ as_validator_validate_web_url (AsValidator *validator, xmlNode *node, const gcha
 		return FALSE;
 	}
 
-	/* do nothing and assume the URL exists if we shouldn't check URLs */
-	if (!priv->check_urls)
-		return TRUE;
-
-	g_debug ("Checking URL: %s\n", url);
-	base_uri = soup_uri_new (url);
-	if (!SOUP_URI_VALID_FOR_HTTP (base_uri)) {
+	if (!as_curl_is_url (url)) {
 		as_validator_add_issue (validator, node, tag,
 					"%s - %s",
 					url,
 					_("URL format is invalid."));
 		return FALSE;
 	}
-	msg = soup_message_new_from_uri (SOUP_METHOD_GET, base_uri);
-	if (msg == NULL) {
-		g_warning ("Failed to setup HTTP GET message for URL.");
-		return FALSE;
-	}
+
+	/* do nothing and assume the URL exists if we shouldn't check URLs */
+	if (!priv->check_urls)
+		return TRUE;
+
+	g_debug ("Checking URL availability: %s\n", url);
 
 	/* try to download the file */
-	status_code = soup_session_send_message (priv->soup_session, msg);
-	if (SOUP_STATUS_IS_TRANSPORT_ERROR (status_code)) {
+	bytes = as_curl_download_bytes (priv->acurl, url, &tmp_error);
+	if (tmp_error != NULL) {
 		as_validator_add_issue (validator, node, tag,
 					"%s - %s",
 					url,
-					soup_status_get_phrase (status_code));
+					tmp_error->message);
 		return FALSE;
-	} else if (status_code != SOUP_STATUS_OK) {
+	} else if (bytes == NULL) {
 		as_validator_add_issue (validator, node, tag,
-					"%s - HTTP %d: %s",
+					"%s - %s",
 					url,
-					status_code, soup_status_get_phrase(status_code));
+					"Unknown error.");
 		return FALSE;
 	}
 
 	/* check if it's a zero sized file */
-	if (msg->response_body->length == 0) {
+	if (g_bytes_get_size (bytes) == 0) {
 		as_validator_add_issue (validator, node, tag,
 					"%s - %s",
 					url,
@@ -392,7 +378,7 @@ as_validator_validate_web_url (AsValidator *validator, xmlNode *node, const gcha
 		return FALSE;
 	}
 
-	/* we we din't get a zero-length file, we just assume everything is fine here */
+	/* if we we din't get a zero-length file, we just assume everything is fine here */
 	return TRUE;
 }
 

src/meson.build

@@ -6,6 +6,7 @@ aslib_src = [
     'as-utils.c',
     # internal
     'as-cache.c',
+    'as-curl.c',
     'as-desktop-entry.c',
     'as-distro-extras.c',
     'as-news-convert.c',
@@ -90,6 +91,7 @@ aslib_priv_headers = [
     'as-component-private.h',
     'as-content-rating-private.h',
     'as-context-private.h',
+    'as-curl.h',
     'as-desktop-entry.h',
     'as-distro-details-private.h',
     'as-distro-extras.h',
@@ -159,7 +161,7 @@ aslib_src = aslib_src + [aslib_gperf_xml, aslib_gperf_yaml]
 aslib_deps = [glib_dep,
               gobject_dep,
               gio_unix_dep,
-              soup_dep,
+              curl_dep,
               lmdb_dep,
               xml2_dep,
               yaml_dep]

tests/ci/install-deps-deb.sh

@@ -28,7 +28,7 @@ eatmydata apt-get install -yq --no-install-recommends \
     libxml2-dev \
     libyaml-dev \
     liblmdb-dev \
-    libsoup2.4-dev \
+    libcurl4-gnutls-dev \
     gtk-doc-tools \
     libgirepository1.0-dev \
     qtbase5-dev \