Commit 124b4c95 authored by Matthias Klumpp's avatar Matthias Klumpp
Browse files

Switch back from Soup to cURL for HTTP(S)

It turns out that people do prefer cURL over libsoup for
minimal/embedded installations, and for some odd reason *do* want
libappstream on those...
Since our use of libsoup is quite light and libappstream is supposed to
not have a lot of heavy dependencies, we just switch back to cURL as
used before. This time though, it's wrapped in a new internal AsCurl
class for convenience.
parent e79a4571
......@@ -49,6 +49,7 @@ you may want to take a look at the [AppStream Generator](https://github.com/ximi
* GObject-Introspection
* libxml2
* libyaml
* libcurl (>= 7.62)
* LMDB
#### Optional
......
......@@ -90,7 +90,7 @@ glib_dep = dependency('glib-2.0', version: '>=2.58')
gobject_dep = dependency('gobject-2.0', version: '>=2.58')
gio_dep = dependency('gio-2.0', version: '>=2.58')
gio_unix_dep = dependency('gio-unix-2.0', version: '>=2.58')
soup_dep = dependency('libsoup-2.4', version: '>= 2.56')
curl_dep = dependency('libcurl', version : '>= 7.62')
xml2_dep = dependency('libxml-2.0')
yaml_dep = dependency('yaml-0.1')
lmdb_dep = dependency('lmdb', required: false)
......
......@@ -11,6 +11,7 @@ src/as-category.c
src/as-checksum.c
src/as-component.c
src/as-content-rating.c
src/as-curl.c
src/as-desktop-entry.c
src/as-distro-details.c
src/as-distro-extras.c
......
/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*-
*
* Copyright (C) 2020-2021 Matthias Klumpp <matthias@tenstral.net>
*
* Licensed under the GNU Lesser General Public License Version 2.1
*
* This library is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 2.1 of the license, or
* (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this library. If not, see <http://www.gnu.org/licenses/>.
*/
/**
* SECTION:as-curl
* @short_description: Internal convenience wrapper around some cURL functions for AppStream
* @include: appstream.h
*/
#include "config.h"
#include "as-curl.h"
#include <curl/curl.h>
struct _AsCurl
{
GObject parent_instance;
};
typedef struct
{
CURL *curl;
const gchar *user_agent;
} AsCurlPrivate;
G_DEFINE_TYPE_WITH_PRIVATE (AsCurl, as_curl, G_TYPE_OBJECT)
#define GET_PRIVATE(o) (as_curl_get_instance_private (o))
G_DEFINE_AUTOPTR_CLEANUP_FUNC(CURLU, curl_url_cleanup)
G_DEFINE_QUARK (AsCurlError, as_curl_error)
/**
* as_curl_is_url:
*
* Check if the URL is valid.
*/
gboolean
as_curl_is_url (const gchar *url)
{
g_autoptr(CURLU) cu = curl_url ();
return curl_url_set (cu, CURLUPART_URL, url, 0) == CURLUE_OK;
}
static void
as_curl_finalize (GObject *object)
{
AsCurl *acurl = AS_CURL (object);
AsCurlPrivate *priv = GET_PRIVATE (acurl);
if (priv->curl != NULL)
curl_easy_cleanup (priv->curl);
G_OBJECT_CLASS (as_curl_parent_class)->finalize (object);
}
static void
as_curl_init (AsCurl *acurl)
{
AsCurlPrivate *priv = GET_PRIVATE (acurl);
priv->user_agent = "appstream/" PACKAGE_VERSION;
}
static void
as_curl_class_init (AsCurlClass *klass)
{
GObjectClass *object_class = G_OBJECT_CLASS (klass);
object_class->finalize = as_curl_finalize;
}
static size_t
as_curl_download_write_bytearray_cb (char *ptr, size_t size, size_t nmemb, void *udata)
{
GByteArray *buf = (GByteArray *) udata;
gsize realsize = size * nmemb;
g_byte_array_append (buf, (const guint8 *) ptr, realsize);
return realsize;
}
/**
* as_curl_download_bytes:
*
* Download an URL as GBytes.
*/
GBytes*
as_curl_download_bytes (AsCurl *acurl, const gchar *url, GError **error)
{
AsCurlPrivate *priv = GET_PRIVATE (acurl);
CURLcode res;
gchar errbuf[CURL_ERROR_SIZE] = { '\0' };
glong status_code = 0;
g_autoptr(GByteArray) buf = g_byte_array_new ();
curl_easy_setopt (priv->curl, CURLOPT_URL, url);
curl_easy_setopt (priv->curl, CURLOPT_ERRORBUFFER, errbuf);
curl_easy_setopt (priv->curl, CURLOPT_WRITEFUNCTION, as_curl_download_write_bytearray_cb);
curl_easy_setopt (priv->curl, CURLOPT_WRITEDATA, buf);
res = curl_easy_perform (priv->curl);
curl_easy_getinfo (priv->curl, CURLINFO_RESPONSE_CODE, &status_code);
if (res != CURLE_OK) {
g_debug ("cURL status-code was %ld", status_code);
if (status_code == 429) {
g_set_error (error,
AS_CURL_ERROR,
AS_CURL_ERROR_REMOTE,
"Failed to download due to server limit");
return NULL;
}
if (errbuf[0] != '\0') {
g_set_error (error,
AS_CURL_ERROR,
AS_CURL_ERROR_DOWNLOAD,
"Failed to download file: %s",
errbuf);
return NULL;
}
g_set_error (error,
AS_CURL_ERROR,
AS_CURL_ERROR_DOWNLOAD,
"Failed to download file: %s",
curl_easy_strerror (res));
return NULL;
}
if (status_code == 404) {
g_set_error (error,
AS_CURL_ERROR,
AS_CURL_ERROR_REMOTE,
"URL was not found on server.");
return NULL;
} else if (status_code != 200) {
g_set_error (error,
AS_CURL_ERROR,
AS_CURL_ERROR_REMOTE,
"Unexpected status code: %ld",
status_code);
return NULL;
}
return g_byte_array_free_to_bytes (g_steal_pointer (&buf));
}
/**
* as_curl_new:
*
* Creates a new #AsCurl.
**/
AsCurl*
as_curl_new (GError **error)
{
AsCurlPrivate *priv;
const gchar *http_proxy;
g_autoptr(AsCurl) acurl = g_object_new (AS_TYPE_CURL, NULL);
priv = GET_PRIVATE (acurl);
priv->curl = curl_easy_init ();
if (priv->curl == NULL) {
g_set_error_literal (error,
AS_CURL_ERROR,
AS_CURL_ERROR_FAILED,
"Failed to setup networking, could not initialize cURL.");
return NULL;
}
if (g_getenv ("AS_CURL_VERBOSE") != NULL)
curl_easy_setopt (priv->curl, CURLOPT_VERBOSE, 1L);
curl_easy_setopt (priv->curl, CURLOPT_USERAGENT, priv->user_agent);
curl_easy_setopt (priv->curl, CURLOPT_CONNECTTIMEOUT, 60L);
curl_easy_setopt (priv->curl, CURLOPT_NOPROGRESS, 1L);
curl_easy_setopt (priv->curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt (priv->curl, CURLOPT_MAXREDIRS, 8L);
/* read common proxy environment variables */
http_proxy = g_getenv ("https_proxy");
if (http_proxy == NULL)
http_proxy = g_getenv ("HTTPS_PROXY");
if (http_proxy == NULL)
http_proxy = g_getenv ("http_proxy");
if (http_proxy == NULL)
http_proxy = g_getenv ("HTTP_PROXY");
if (http_proxy != NULL && strlen (http_proxy) > 0)
curl_easy_setopt (priv->curl, CURLOPT_PROXY, http_proxy);
return AS_CURL (g_steal_pointer (&acurl));
}
/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*-
*
* Copyright (C) 2020-2021 Matthias Klumpp <matthias@tenstral.net>
*
* Licensed under the GNU Lesser General Public License Version 2.1
*
* This library is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 2.1 of the license, or
* (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this library. If not, see <http://www.gnu.org/licenses/>.
*/
#if !defined (AS_COMPILATION)
#error "Can not use internal AppStream API from external project."
#endif
#pragma once
#include <glib-object.h>
G_BEGIN_DECLS
#define AS_TYPE_CURL (as_curl_get_type ())
G_DECLARE_FINAL_TYPE (AsCurl, as_curl, AS, CURL, GObject)
/**
* AsCurlError:
* @AS_CURL_ERROR_FAILED: Generic failure.
* @AS_CURL_ERROR_REMOTE: Some issue happened on the remote side.
* @AS_CURL_ERROR_DOWNLOAD: Download failed.
*
* An cURL error.
**/
typedef enum {
AS_CURL_ERROR_FAILED,
AS_CURL_ERROR_REMOTE,
AS_CURL_ERROR_DOWNLOAD,
/*< private >*/
AS_CURL_ERROR_LAST
} AsCurlError;
#define AS_CURL_ERROR as_curl_error_quark ()
GQuark as_curl_error_quark (void);
AsCurl *as_curl_new (GError **error);
GBytes *as_curl_download_bytes (AsCurl *acurl,
const gchar *url,
GError **error);
gboolean as_curl_is_url (const gchar *url);
G_END_DECLS
......@@ -36,15 +36,13 @@
#include <libxml/parser.h>
#include <string.h>
#include <libsoup/soup.h>
#include <libsoup/soup-status.h>
#include "as-validator.h"
#include "as-validator-issue.h"
#include "as-validator-issue-tag.h"
#include "as-utils.h"
#include "as-utils-private.h"
#include "as-curl.h"
#include "as-vercmp.h"
#include "as-spdx.h"
#include "as-component.h"
......@@ -62,7 +60,7 @@ typedef struct
gchar *current_fname;
gboolean check_urls;
SoupSession *soup_session;
AsCurl *acurl;
} AsValidatorPrivate;
G_DEFINE_TYPE_WITH_PRIVATE (AsValidator, as_validator, G_TYPE_OBJECT)
......@@ -99,7 +97,6 @@ as_validator_init (AsValidator *validator)
g_str_equal,
g_free,
(GDestroyNotify) g_ptr_array_unref);
priv->soup_session = NULL;
priv->current_fname = NULL;
priv->current_cpt = NULL;
priv->check_urls = FALSE;
......@@ -123,8 +120,8 @@ as_validator_finalize (GObject *object)
if (priv->current_cpt != NULL)
g_object_unref (priv->current_cpt);
if (priv->soup_session != NULL)
g_object_unref (priv->soup_session);
if (priv->acurl != NULL)
g_object_unref (priv->acurl);
G_OBJECT_CLASS (as_validator_parent_class)->finalize (object);
}
......@@ -298,26 +295,21 @@ static gboolean
as_validator_setup_networking (AsValidator *validator)
{
AsValidatorPrivate *priv = GET_PRIVATE (validator);
g_autoptr(GError) tmp_error = NULL;
/* check if we are already initialized */
if (priv->soup_session != NULL)
if (priv->acurl != NULL)
return TRUE;
/* don't initialize if no URLs should be checked */
if (!priv->check_urls)
return TRUE;
priv->soup_session = soup_session_new_with_options (SOUP_SESSION_USER_AGENT,
"appstream-validator",
SOUP_SESSION_TIMEOUT,
90,
NULL);
if (priv->soup_session == NULL) {
g_critical ("Failed to set up networking support");
priv->acurl = as_curl_new (&tmp_error);
if (priv->acurl == NULL) {
g_critical ("Failed to set up networking support: %s", tmp_error->message);
return FALSE;
}
soup_session_add_feature_by_type (priv->soup_session,
SOUP_TYPE_PROXY_RESOLVER_DEFAULT);
return TRUE;
}
......@@ -331,9 +323,8 @@ static gboolean
as_validator_validate_web_url (AsValidator *validator, xmlNode *node, const gchar *url, const gchar *tag)
{
AsValidatorPrivate *priv = GET_PRIVATE (validator);
guint status_code;
g_autoptr(SoupMessage) msg = NULL;
g_autoptr(SoupURI) base_uri = NULL;
g_autoptr(GBytes) bytes = NULL;
g_autoptr(GError) tmp_error = NULL;
/* we don't check mailto URLs */
if (g_str_has_prefix (url, "mailto:"))
......@@ -347,43 +338,38 @@ as_validator_validate_web_url (AsValidator *validator, xmlNode *node, const gcha
return FALSE;
}
/* do nothing and assume the URL exists if we shouldn't check URLs */
if (!priv->check_urls)
return TRUE;
g_debug ("Checking URL: %s\n", url);
base_uri = soup_uri_new (url);
if (!SOUP_URI_VALID_FOR_HTTP (base_uri)) {
if (!as_curl_is_url (url)) {
as_validator_add_issue (validator, node, tag,
"%s - %s",
url,
_("URL format is invalid."));
return FALSE;
}
msg = soup_message_new_from_uri (SOUP_METHOD_GET, base_uri);
if (msg == NULL) {
g_warning ("Failed to setup HTTP GET message for URL.");
return FALSE;
}
/* do nothing and assume the URL exists if we shouldn't check URLs */
if (!priv->check_urls)
return TRUE;
g_debug ("Checking URL availability: %s\n", url);
/* try to download the file */
status_code = soup_session_send_message (priv->soup_session, msg);
if (SOUP_STATUS_IS_TRANSPORT_ERROR (status_code)) {
bytes = as_curl_download_bytes (priv->acurl, url, &tmp_error);
if (tmp_error != NULL) {
as_validator_add_issue (validator, node, tag,
"%s - %s",
url,
soup_status_get_phrase (status_code));
tmp_error->message);
return FALSE;
} else if (status_code != SOUP_STATUS_OK) {
} else if (bytes == NULL) {
as_validator_add_issue (validator, node, tag,
"%s - HTTP %d: %s",
"%s - %s",
url,
status_code, soup_status_get_phrase(status_code));
"Unknown error.");
return FALSE;
}
/* check if it's a zero sized file */
if (msg->response_body->length == 0) {
if (g_bytes_get_size (bytes) == 0) {
as_validator_add_issue (validator, node, tag,
"%s - %s",
url,
......@@ -392,7 +378,7 @@ as_validator_validate_web_url (AsValidator *validator, xmlNode *node, const gcha
return FALSE;
}
/* we we din't get a zero-length file, we just assume everything is fine here */
/* if we we din't get a zero-length file, we just assume everything is fine here */
return TRUE;
}
......
......@@ -6,6 +6,7 @@ aslib_src = [
'as-utils.c',
# internal
'as-cache.c',
'as-curl.c',
'as-desktop-entry.c',
'as-distro-extras.c',
'as-news-convert.c',
......@@ -90,6 +91,7 @@ aslib_priv_headers = [
'as-component-private.h',
'as-content-rating-private.h',
'as-context-private.h',
'as-curl.h',
'as-desktop-entry.h',
'as-distro-details-private.h',
'as-distro-extras.h',
......@@ -159,7 +161,7 @@ aslib_src = aslib_src + [aslib_gperf_xml, aslib_gperf_yaml]
aslib_deps = [glib_dep,
gobject_dep,
gio_unix_dep,
soup_dep,
curl_dep,
lmdb_dep,
xml2_dep,
yaml_dep]
......
......@@ -28,7 +28,7 @@ eatmydata apt-get install -yq --no-install-recommends \
libxml2-dev \
libyaml-dev \
liblmdb-dev \
libsoup2.4-dev \
libcurl4-gnutls-dev \
gtk-doc-tools \
libgirepository1.0-dev \
qtbase5-dev \
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment