From a8186ed96125f93b8963a8265ac42f56a837fb2b Mon Sep 17 00:00:00 2001 From: Matthias Klumpp Date: Fri, 19 Nov 2021 05:19:59 +0100 Subject: [PATCH] Refactor the caching code and partition cache into sections The new cache is now backed by xmlb instead of an LMDB database, which allows us to perform a lot more complex queries with low effort. The cache is also now shared between all applications by default (by popular request), for every grouping of metadata. In addition to that, what AsPool understands as "cache" is now a collection of partitions, called sections, which represent AppStream metadata from one domain, e.g. one Flatpak repository, the OS' collection metadata, the combined metainfo/desktop-entry data of the system, etc. This permits updating those sections independently, which means that if a MetaInfo file changes, we will not have to rebuild the whole cache, but only a small section of it. This is a prerequisite for efficient monitoring of metadata directories, and a lot of other neat optimizations. Since AppStream is used in desktop shells nowadays, support for this is a needed addition to not keep the system busy with needless work and cause lag. In addition to that, a lot of cruft and complex code has also been cleaned up. The current code runs about 60% slower than the previous cache on cache rebuilds, query time is about 10% slower. There is a lot of room for improvements though, and we will likely get to the previous times before release. Caution! The new code is not yet fully threadsafe and has various rough edges, but it compiles and passes the testsuite. Further improvements are located in smaller, easier to manage follow-up patches. CC: #337 --- docs/API-TODO.md | 4 + meson.build | 9 +- qt/pool.cpp | 17 +- qt/pool.h | 17 +- qt/tests/asqt-pool-test.cpp | 5 +- src/as-cache.c | 3534 ++++++++++++----------------------- src/as-cache.h | 199 +- src/as-component.c | 45 +- src/as-distro-extras.c | 2 +- src/as-pool-private.h | 14 +- src/as-pool.c | 1788 ++++++++---------- src/as-pool.h | 56 +- src/as-settings-private.h | 2 +- src/as-tag-xml.gperf | 1 + src/as-tag.h | 1 + src/as-utils-private.h | 3 + src/as-utils.c | 18 + src/as-utils.h | 2 + src/as-xml.c | 2 + src/meson.build | 5 +- tests/test-performance.c | 75 +- tests/test-pool.c | 63 +- tools/appstreamcli.c | 12 +- tools/ascli-actions-mdata.c | 51 +- 24 files changed, 2405 insertions(+), 3520 deletions(-) diff --git a/docs/API-TODO.md b/docs/API-TODO.md index 587764ab0..51cbc8ae9 100644 --- a/docs/API-TODO.md +++ b/docs/API-TODO.md @@ -23,3 +23,7 @@ libappstream API break for the AppStream 1.0 release. (there are likely some performance improvements to be found there) * Make UNKNOWN the first entry in AsFormatVersion enum + + * Rename AsPoolFlags: e.g. AS_POOL_FLAG_READ_COLLECTION -> AS_POOL_FLAG_USE_OS_COLLECTION + + * Cleanup AsPool API, only keep sensible functions (maybe make the pool read-only?) diff --git a/meson.build b/meson.build index f9f8f7e3a..6dc656b9c 100644 --- a/meson.build +++ b/meson.build @@ -93,14 +93,7 @@ gio_unix_dep = dependency('gio-unix-2.0', version: '>=2.58') curl_dep = dependency('libcurl', version : '>= 7.62') xml2_dep = dependency('libxml-2.0') yaml_dep = dependency('yaml-0.1') -lmdb_dep = dependency('lmdb', required: false) - -if not lmdb_dep.found() - lmdb_lib = cc.find_library('lmdb', required: true) - if not cc.has_header('lmdb.h') - error('Headers for dependency "lmdb" not found') - endif -endif +xmlb_dep = dependency('xmlb', version : '>= 0.3.2') if get_option ('gir') # ensure we have a version of GIR that isn't broken with Meson diff --git a/qt/pool.cpp b/qt/pool.cpp index 5ada19c86..ef074df91 100644 --- a/qt/pool.cpp +++ b/qt/pool.cpp @@ -18,6 +18,7 @@ */ #include +#include "as-pool-private.h" #include "pool.h" #include @@ -151,12 +152,16 @@ QList Pool::search(const QString& term) const void Pool::clearMetadataLocations() { +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" as_pool_clear_metadata_locations(d->pool); +#pragma GCC diagnostic pop } void Pool::addMetadataLocation(const QString& directory) { +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" as_pool_add_metadata_location (d->pool, qPrintable(directory)); +#pragma GCC diagnostic pop } void Pool::setLocale(const QString& locale) @@ -174,22 +179,30 @@ void Pool::setFlags(uint flags) as_pool_set_flags (d->pool, (AsPoolFlags) flags); } +void Pool::overrideCacheLocations(const QString &sysDir, const QString &userDir) +{ + as_pool_override_cache_locations (d->pool, qPrintable(sysDir), qPrintable(userDir)); +} + uint Pool::cacheFlags() const { - return (uint) as_pool_get_cache_flags(d->pool); + return 0; } void Pool::setCacheFlags(uint flags) { - as_pool_set_cache_flags (d->pool, (AsCacheFlags) flags); } QString AppStream::Pool::cacheLocation() const { +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" return QString::fromUtf8(as_pool_get_cache_location(d->pool)); +#pragma GCC diagnostic pop } void Pool::setCacheLocation(const QString &location) { +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" as_pool_set_cache_location(d->pool, qPrintable(location)); +#pragma GCC diagnostic pop } diff --git a/qt/pool.h b/qt/pool.h index 6d8d54aa9..ccf63e27b 100644 --- a/qt/pool.h +++ b/qt/pool.h @@ -46,7 +46,9 @@ Q_OBJECT * FlagNone: No flags. * FlagReadCollection: Add AppStream collection metadata to the pool. * FlagReadMetainfo: Add data from AppStream metainfo files to the pool. - * FlagReadDesktopFiles: Add metadata from .desktop files to the pool. + * FlagReadDesktopFiles: Add metadata from desktop-entry files to the pool. + * FlagLoadFlatpak: Add AppStream metadata from Flatpak to the pool. + * FlagIgnoreCacheAge: Ignore cache age and always load data from scratch. * * Flags on how caching should be used. **/ @@ -55,6 +57,8 @@ Q_OBJECT FlagReadCollection = 1 << 0, FlagReadMetainfo = 1 << 1, FlagReadDesktopFiles = 1 << 2, + FlagLoadFlatpak = 1 << 3, + FlagIgnoreCacheAge = 1 << 4, }; /** @@ -123,11 +127,14 @@ Q_OBJECT uint flags() const; void setFlags(uint flags); - uint cacheFlags() const; - void setCacheFlags(uint flags); + void overrideCacheLocations(const QString &sysDir, + const QString &userDir); - void setCacheLocation(const QString &path); - QString cacheLocation() const; + Q_DECL_DEPRECATED uint cacheFlags() const; + Q_DECL_DEPRECATED void setCacheFlags(uint flags); + + Q_DECL_DEPRECATED void setCacheLocation(const QString &path); + Q_DECL_DEPRECATED QString cacheLocation() const; private: Q_DISABLE_COPY(Pool); diff --git a/qt/tests/asqt-pool-test.cpp b/qt/tests/asqt-pool-test.cpp index 7852f03fc..8b1994795 100644 --- a/qt/tests/asqt-pool-test.cpp +++ b/qt/tests/asqt-pool-test.cpp @@ -44,10 +44,11 @@ void PoolReadTest::testRead01() auto flags = pool->flags(); flags &= ~Pool::FlagReadDesktopFiles; flags &= ~Pool::FlagReadMetainfo; + flags &= ~Pool::FlagIgnoreCacheAge; pool->setFlags(flags); - // don't use caches - pool->setCacheFlags(Pool::CacheFlagNone); + // use clean caches + pool->overrideCacheLocations("/tmp", nullptr); // read metadata QVERIFY(pool->load()); diff --git a/src/as-cache.c b/src/as-cache.c index 61fcaeec3..9eb2238d9 100644 --- a/src/as-cache.c +++ b/src/as-cache.c @@ -24,7 +24,6 @@ * * Caches are used by #AsPool to quickly search for components while not keeping all * component data in memory. - * Internally, a cache is backed by an LMDB database. * This class is threadsafe. * * See also: #AsPool @@ -33,104 +32,114 @@ #include "config.h" #include "as-cache.h" -#include #include #include #include #include +#include +#include #include "as-utils-private.h" #include "as-context-private.h" #include "as-component-private.h" #include "as-launchable.h" -/* - * The maximum size of the cache file (128MiB on x86, - * 256MiB everywhere else). - * The file is mmap(2)'d into memory. - */ -#ifdef __i386__ -static const size_t LMDB_DB_SIZE = 1024 * 1024 * 128; -#else -static const size_t LMDB_DB_SIZE = 1024 * 1024 * 256; -#endif -/* format version of the currently supported cache */ -static const gchar *CACHE_FORMAT_VERSION = "1"; +typedef struct +{ + gchar *cache_root_dir; + gchar *system_cache_dir; + GRefString *locale; + gboolean default_paths_changed; -/* checksum algorithm used in the cache */ -#define AS_CACHE_CHECKSUM G_CHECKSUM_MD5 + AsContext *context; + GPtrArray *sections; + GHashTable *masked; -/* digest length of AS_CACHE_CHECKSUM */ -static const gsize AS_CACHE_CHECKSUM_LEN = 16; + AsCacheDataRefineFn cpt_refine_func; + gboolean prefer_os_metainfo; -typedef struct -{ - MDB_env *db_env; - MDB_dbi db_cpts; - MDB_dbi db_cids; - MDB_dbi db_fts; - MDB_dbi db_cats; - MDB_dbi db_launchables; - MDB_dbi db_provides; - MDB_dbi db_kinds; - MDB_dbi db_addons; - - gchar *fname; - gchar *volatile_db_fname; - gsize max_keysize; - gboolean opened; - gboolean nosync; - gboolean readonly; - - AsContext *context; - gchar *locale; - - gboolean floating; - GHashTable *cpt_map; - GHashTable *cid_set; - GHashTable *ro_removed_set; - - GFunc cpt_refine_func; - gpointer cpt_refine_func_udata; - - GMutex mutex; + GMutex sec_mutex; + GMutex mask_mutex; } AsCachePrivate; G_DEFINE_TYPE_WITH_PRIVATE (AsCache, as_cache, G_TYPE_OBJECT) #define GET_PRIVATE(o) (as_cache_get_instance_private (o)) -static gboolean as_cache_register_addons_for_component (AsCache *cache, - MDB_txn *txn, - AsComponent *cpt, - GError **error); - -/** - * as_cache_checksum_hash: - * - * Hash cache checksums for use in a hash table. - */ -guint -as_cache_checksum_hash (gconstpointer v) +typedef struct { + gboolean is_os_data; + gboolean is_mask; + gchar *key; + AsComponentScope scope; + AsFormatStyle format_style; + XbSilo *silo; + gchar *fname; + + gpointer refine_func_udata; +} AsCacheSection; + +static AsCacheSection* +as_cache_section_new (const gchar *key) { - guint32 h = 5381; - const guint8 *p = v; + AsCacheSection *csec; + csec = g_new0 (AsCacheSection, 1); + csec->format_style = AS_FORMAT_STYLE_COLLECTION; + csec->key = g_strdup (key); + return csec; +} - for (gsize i = 0; i < AS_CACHE_CHECKSUM_LEN; i++) - h = (h << 5) + h + p[i]; +static void +as_cache_section_free (AsCacheSection *csec) +{ + g_free (csec->key); + g_free (csec->fname); + if (csec->silo != NULL) + g_object_unref (csec->silo); + g_free (csec); +} - return h; +gint +as_cache_section_cmp (gconstpointer a, gconstpointer b) +{ + const AsCacheSection *s1 = a; + const AsCacheSection *s2 = b; + + /* sort masking data last */ + if (s1->is_mask && !s2->is_mask) + return 1; + if (!s1->is_mask && s2->is_mask) + return -1; + + /* sort collection metadata before metainfo data */ + if (s1->format_style > s2->format_style) + return -1; + if (s1->format_style < s2->format_style) + return 1; + + /* system scope before user scope */ + if (s1->scope < s2->scope) + return -1; + if (s1->scope > s2->scope) + return 1; + + /* alphabetic sorting if everything else is equal */ + return g_strcmp0 (s1->key, s2->key); } +G_DEFINE_AUTOPTR_CLEANUP_FUNC(AsCacheSection, as_cache_section_free) + /** - * as_cache_checksum_equal: + * as_cache_error_quark: * - * Equality check for two cache checksums. - */ -gboolean -as_cache_checksum_equal (gconstpointer v1, gconstpointer v2) + * Return value: An error quark. + **/ +GQuark +as_cache_error_quark (void) { - return memcmp (v1, v2, AS_CACHE_CHECKSUM_LEN) == 0; + static GQuark quark = 0; + if (!quark) + quark = g_quark_from_static_string ("AsCache"); + return quark; } /** @@ -141,38 +150,32 @@ as_cache_init (AsCache *cache) { AsCachePrivate *priv = GET_PRIVATE (cache); - g_mutex_init (&priv->mutex); + g_mutex_init (&priv->sec_mutex); + g_mutex_init (&priv->mask_mutex); + + priv->sections = g_ptr_array_new_with_free_func ((GDestroyNotify) as_cache_section_free); + priv->masked = g_hash_table_new_full (g_str_hash, + g_str_equal, + g_free, + NULL); - priv->opened = FALSE; - priv->max_keysize = 511; - priv->cpt_refine_func = NULL; - priv->locale = as_get_current_locale (); priv->context = as_context_new (); - as_context_set_locale (priv->context, priv->locale); as_context_set_style (priv->context, AS_FORMAT_STYLE_COLLECTION); - as_context_set_internal_mode (priv->context, TRUE); /* we need to write some special tags only used in the cache */ + /* we need to write some special tags only used in the cache */ + as_context_set_internal_mode (priv->context, TRUE); - priv->floating = FALSE; - priv->readonly = FALSE; + /* default string for unknown locale - should be replaced before use */ + as_cache_set_locale (cache, "unknown"); - /* stores known components in-memory for faster access */ - priv->cpt_map = g_hash_table_new_full (g_str_hash, - g_str_equal, - g_free, - (GDestroyNotify) g_object_unref); - - /* set which stores whether we have seen a component-ID already */ - priv->cid_set = g_hash_table_new_full (g_str_hash, - g_str_equal, - g_free, - NULL); - - /* set of removed component hashes in a read-only cache */ - priv->ro_removed_set = g_hash_table_new_full (as_cache_checksum_hash, - as_cache_checksum_equal, - g_free, - NULL); + /* we choose the system-wide cache directory if we are root, otherwise we will write to + * a user-specific directory. The system cache dir is always considered immutable. */ + priv->system_cache_dir = g_strdup (AS_APPSTREAM_SYS_CACHE_DIR); + if (as_utils_is_root ()) + priv->cache_root_dir = g_strdup (priv->system_cache_dir); + else + priv->cache_root_dir = as_get_user_cache_dir (); + priv->default_paths_changed = FALSE; } /** @@ -184,19 +187,15 @@ as_cache_finalize (GObject *object) AsCache *cache = AS_CACHE (object); AsCachePrivate *priv = GET_PRIVATE (cache); - as_cache_close (cache); - g_mutex_lock (&priv->mutex); + g_free (priv->cache_root_dir); + g_free (priv->system_cache_dir); + as_ref_string_release (priv->locale); g_object_unref (priv->context); - g_free (priv->locale); - g_free (priv->fname); - - g_hash_table_unref (priv->cpt_map); - g_hash_table_unref (priv->cid_set); - g_hash_table_unref (priv->ro_removed_set); - - g_mutex_unlock (&priv->mutex); - g_mutex_clear (&priv->mutex); + g_ptr_array_unref (priv->sections); + g_hash_table_unref (priv->masked); + g_mutex_clear (&priv->sec_mutex); + g_mutex_clear (&priv->mask_mutex); G_OBJECT_CLASS (as_cache_parent_class)->finalize (object); } @@ -211,2480 +210,1375 @@ as_cache_class_init (AsCacheClass *klass) } /** - * as_generate_cache_checksum: + * as_cache_new: * - * Create checksum digest used internally in the database. - */ -inline static void -as_generate_cache_checksum (const gchar *value, gssize value_len, guint8 **result, gsize *result_len) -{ - gsize rlen = AS_CACHE_CHECKSUM_LEN; /* digest length (16 for MD5) */ - g_autoptr(GChecksum) cs = g_checksum_new (AS_CACHE_CHECKSUM); - - *result = g_malloc (sizeof(guint8) * rlen); - g_checksum_update (cs, (const guchar *) value, value_len); - g_checksum_get_digest (cs, *result, &rlen); - - if (result_len != NULL) - *result_len = rlen; -} - -/** - * as_cache_open_subdb: + * Creates a new #AsCache. * - * helper for %as_cache_open() - */ -static gboolean -as_cache_open_subdb (MDB_txn *txn, const gchar *name, MDB_dbi *dbi, GError **error) + * Returns: (transfer full): a #AsCache + * + **/ +AsCache* +as_cache_new (void) { - gint rc; - rc = mdb_dbi_open (txn, name, MDB_CREATE, dbi); - if (rc != MDB_SUCCESS) { - g_set_error (error, - AS_CACHE_ERROR, - AS_CACHE_ERROR_FAILED, - "Unable to read %s database: %s", name, mdb_strerror (rc)); - return FALSE; - } - return TRUE; + AsCache *cache; + cache = g_object_new (AS_TYPE_CACHE, NULL); + return AS_CACHE (cache); } /** - * as_cache_transaction_new: + * as_cache_get_locale: + * @cache: an #AsCache instance. + * + * Gets the locale this cache is generated for. + * + * Returns: locale string, or "unknown" if unset + * + * Since: 0.14.0 */ -static MDB_txn* -as_cache_transaction_new (AsCache *cache, guint flags, GError **error) +const gchar* +as_cache_get_locale (AsCache *cache) { AsCachePrivate *priv = GET_PRIVATE (cache); - gint rc; - MDB_txn *txn; - - rc = mdb_txn_begin (priv->db_env, NULL, flags, &txn); - if (rc != MDB_SUCCESS) { - g_set_error (error, - AS_CACHE_ERROR, - AS_CACHE_ERROR_FAILED, - "Unable to create transaction: %s", mdb_strerror (rc)); - return NULL; - } - - return txn; + return priv->locale; } /** - * lmdb_transaction_commit: + * as_cache_set_locale: + * @cache: an #AsCache instance. + * @locale: the locale name. + * + * Set the cache locale. */ -static gboolean -lmdb_transaction_commit (MDB_txn *txn, GError **error) +void +as_cache_set_locale (AsCache *cache, const gchar *locale) { - gint rc; - rc = mdb_txn_commit (txn); - if (rc != MDB_SUCCESS) { - g_set_error (error, - AS_CACHE_ERROR, - AS_CACHE_ERROR_FAILED, - "Unable to commit transaction: %s", mdb_strerror (rc)); - return FALSE; - } - - return TRUE; + AsCachePrivate *priv = GET_PRIVATE (cache); + as_ref_string_assign_safe (&priv->locale, locale); + as_context_set_locale (priv->context, priv->locale); } /** - * lmdb_transaction_abort: + * as_cache_set_locations: + * @cache: an #AsCache instance. + * @system_cache_dir: System cache directory + * @user_cache_dir: User cache directory + * + * Override the default cache locations. The system cache is only written to if + * the application is running with root permissions (in which case it is set equal + * to the user cache directory), otherwise it is used to load caches that are up-to-date + * but not available in the user directory yet. + * + * The user directory is generally writable and will store arbitrary caches for any applications. + * + * You should *not* need to call this function! It should generally only be used for debugging + * as well as tests. */ -static void -lmdb_transaction_abort (MDB_txn *txn) +void +as_cache_set_locations (AsCache *cache, const gchar *system_cache_dir, const gchar *user_cache_dir) { - if (txn == NULL) - return; - mdb_txn_abort (txn); + AsCachePrivate *priv = GET_PRIVATE (cache); + g_free (priv->cache_root_dir); + priv->cache_root_dir = g_strdup (user_cache_dir); + g_free (priv->system_cache_dir); + priv->system_cache_dir = g_strdup (system_cache_dir); + priv->default_paths_changed = TRUE; } /** - * lmdb_str_to_dbval: + * as_cache_get_prefer_os_metainfo: + * @cache: an #AsCache instance. + * + * Get whether metainfo data should be preferred over collection data for OS components. * - * Get MDB_val for string. + * Returns: %TRUE if metainfo data is preferred. */ -inline static MDB_val -lmdb_str_to_dbval (const gchar *data, gssize len) +gboolean +as_cache_get_prefer_os_metainfo (AsCache *cache) { - MDB_val val; - if (len < 0) - len = sizeof(gchar) * strlen (data); - - val.mv_size = len; - val.mv_data = (void*) data; - return val; + AsCachePrivate *priv = GET_PRIVATE (cache); + return priv->prefer_os_metainfo; } /** - * lmdb_val_strdup: + * as_cache_set_prefer_os_metainfo: + * @cache: an #AsCache instance. + * @prefer_os_metainfo: Whether Metainfo data is preferred. * - * Get string from database value. + * Set whether metainfo data should be preferred over collection data for OS components. */ -inline static gchar* -lmdb_val_strdup (MDB_val val) +void +as_cache_set_prefer_os_metainfo (AsCache *cache, gboolean prefer_os_metainfo) { - if (val.mv_size == 0) - return NULL; - return g_strndup (val.mv_data, val.mv_size); + AsCachePrivate *priv = GET_PRIVATE (cache); + priv->prefer_os_metainfo = prefer_os_metainfo; } /** - * lmdb_val_memdup: - * - * Duplicate memory chunk from database value. + * as_cache_delete_file_if_old: */ -inline static void* -lmdb_val_memdup (MDB_val val, gsize *len) +static void +as_delete_cache_file_if_old (AsCache *cache, const gchar *fname, time_t min_atime) { - *len = val.mv_size; - if (val.mv_size == 0) - return NULL; -#if GLIB_CHECK_VERSION(2,68,0) - return g_memdup2 (val.mv_data, val.mv_size); -#else - return g_memdup (val.mv_data, val.mv_size); -#endif + struct stat sbuf; + + /* safeguard so we will only delete cache files */ + if (!g_str_has_suffix (fname, ".xb") && + !g_str_has_suffix (fname, ".cache")) + return; + + if (stat (fname, &sbuf) < 0) + return; + + if (sbuf.st_atime < min_atime) { + g_autofree gchar *basename = g_path_get_basename (fname); + g_debug ("Deleting old cache file: %s", basename); + g_unlink (fname); + } } /** - * as_cache_txn_put_kv: - * - * Add key/value pair to the database + * as_cache_remove_old_data_from_dir: */ -static gboolean -as_cache_txn_put_kv (AsCache *cache, MDB_txn *txn, MDB_dbi dbi, const gchar *key, MDB_val dval, GError **error) +static void +as_cache_remove_old_data_from_dir (AsCache *cache, const gchar *cache_dir) { - AsCachePrivate *priv = GET_PRIVATE (cache); - MDB_val dkey; - gint rc; - gsize key_len; - g_autofree gchar *key_hash = NULL; - - /* if key is too short, return error */ - g_assert (key != NULL); - key_len = sizeof(gchar) * strlen (key); - if (key_len == 0) { - g_set_error (error, - AS_CACHE_ERROR, - AS_CACHE_ERROR_BAD_DATA, - "Can not add an empty (zero-length) key to the cache"); - return FALSE; + g_autoptr(GFileEnumerator) direnum = NULL; + g_autoptr(GFile) cdir = NULL; + g_autoptr(GError) error = NULL; + time_t min_last_atime; + + cdir = g_file_new_for_path (cache_dir); + direnum = g_file_enumerate_children (cdir, + G_FILE_ATTRIBUTE_STANDARD_NAME, + G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, + NULL, + &error); + if (error != NULL) { + g_debug ("Unable to clean cache directory '%s': %s", + cache_dir, error->message); + return; } - /* if key is too long, hash it */ - if (key_len > priv->max_keysize) { - key_hash = g_compute_checksum_for_string (AS_CACHE_CHECKSUM, key, key_len); - dkey = lmdb_str_to_dbval (key_hash, -1); - } else { - dkey = lmdb_str_to_dbval (key, key_len); - } + min_last_atime = time (NULL) - (3 * 30 * 24 * 60 * 60); + while (TRUE) { + GFileInfo *finfo = NULL; + g_autofree gchar *fname_full = NULL; - rc = mdb_put (txn, dbi, &dkey, &dval, 0); - if (rc != MDB_SUCCESS) { - g_set_error (error, - AS_CACHE_ERROR, - AS_CACHE_ERROR_FAILED, - "Unable to add data: %s", mdb_strerror (rc)); - return FALSE; - } + if (!g_file_enumerator_iterate (direnum, &finfo, NULL, NULL, NULL)) + return; + if (finfo == NULL) + break; - return TRUE; + /* generate full filename */ + fname_full = g_build_filename (cache_dir, + g_file_info_get_name (finfo), + NULL); + + /* jump one directory deeper */ + if (g_file_info_get_file_type (finfo) == G_FILE_TYPE_DIRECTORY) { + g_autoptr(GFileEnumerator) sd_direnum = NULL; + g_autoptr(GFile) sd_cdir = NULL; + + sd_cdir = g_file_new_for_path (fname_full); + sd_direnum = g_file_enumerate_children (sd_cdir, + G_FILE_ATTRIBUTE_STANDARD_NAME, + G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, + NULL, NULL); + if (sd_direnum == NULL) { + g_debug ("Unable to scan directory '%s'.", fname_full); + continue; + } + + while (TRUE) { + GFileInfo *sd_finfo = NULL; + g_autofree gchar *sd_fname_full = NULL; + if (!g_file_enumerator_iterate (sd_direnum, &sd_finfo, NULL, NULL, NULL)) + break; + if (sd_finfo == NULL) + break; + if (g_file_info_get_file_type (sd_finfo) != G_FILE_TYPE_REGULAR) + continue; + + sd_fname_full = g_build_filename (fname_full, + g_file_info_get_name (sd_finfo), + NULL); + as_delete_cache_file_if_old (cache, sd_fname_full, min_last_atime); + } + + /* just try to delete the directory - if it's not yet empty, nothing will happen */ + g_remove (fname_full); + } + if (g_file_info_get_file_type (finfo) != G_FILE_TYPE_REGULAR) + continue; + as_delete_cache_file_if_old (cache, fname_full, min_last_atime); + } } /** - * as_cache_txn_put_kv_str: + * as_cache_prune_data: + * @cache: an #AsCache instance. * - * Add key/value pair to the database + * Delete all cache data that has not been used in a long time. */ -static gboolean -as_cache_txn_put_kv_str (AsCache *cache, MDB_txn *txn, MDB_dbi dbi, const gchar *key, const gchar *value, GError **error) +void +as_cache_prune_data (AsCache *cache) { - MDB_val dvalue; - dvalue = lmdb_str_to_dbval (value, -1); + AsCachePrivate *priv = GET_PRIVATE (cache); + + if (priv->default_paths_changed) { + /* we will not delete stuff from non-default cache locations, to prevent accidental data loss */ + g_debug ("Not pruning cache: Default paths were changed."); + return; + } - return as_cache_txn_put_kv (cache, txn, dbi, key, dvalue, error); + g_debug ("Pruning old cache data."); + as_cache_remove_old_data_from_dir (cache, priv->cache_root_dir); + if (as_utils_is_writable (priv->system_cache_dir)) + as_cache_remove_old_data_from_dir (cache, priv->system_cache_dir); } /** - * as_cache_txn_put_kv_bytes: + * as_cache_clear: + * @cache: an #AsCache instance. * - * Add string key / byte value pair to the database + * Close cache and reset all opened cache sections. + * Also forget all masked components, but keep any settings. */ -static gboolean -as_cache_txn_put_kv_bytes (AsCache *cache, MDB_txn *txn, MDB_dbi dbi, const gchar *key, - const guint8 *value, gsize value_len, GError **error) +void +as_cache_clear (AsCache *cache) { - MDB_val dval; - dval.mv_data = (guint8*) value; - dval.mv_size = value_len; + AsCachePrivate *priv = GET_PRIVATE (cache); + g_autoptr(GMutexLocker) locker1 = g_mutex_locker_new (&priv->sec_mutex); + g_autoptr(GMutexLocker) locker2 = g_mutex_locker_new (&priv->mask_mutex); + + g_ptr_array_set_size (priv->sections, 0); - return as_cache_txn_put_kv (cache, txn, dbi, key, dval, error); + g_hash_table_unref (priv->masked); + priv->masked = g_hash_table_new_full (g_str_hash, + g_str_equal, + g_free, + NULL); } /** - * as_txn_put_hash_kv_str: - * - * Add checksum key / string value pair to the database + * as_cache_get_root_dir_with_scope: */ -static gboolean -as_txn_put_hash_kv_str (MDB_txn *txn, MDB_dbi dbi, const guint8 *hash, const gchar *value, GError **error) +static gchar * +as_cache_get_root_dir_with_scope (AsCache *cache, AsCacheScope cache_scope, AsComponentScope scope) { - MDB_val dkey; - MDB_val dval; - gint rc; + AsCachePrivate *priv = GET_PRIVATE (cache); + gchar *path; + const gchar *cache_root; - dkey.mv_size = AS_CACHE_CHECKSUM_LEN; - dkey.mv_data = (guint8*) hash; - dval = lmdb_str_to_dbval (value, -1); + if (cache_scope == AS_CACHE_SCOPE_SYSTEM) + cache_root = priv->system_cache_dir; + else + cache_root = priv->cache_root_dir; - rc = mdb_put (txn, dbi, &dkey, &dval, 0); - if (rc != MDB_SUCCESS) { - g_set_error (error, - AS_CACHE_ERROR, - AS_CACHE_ERROR_FAILED, - "Unable to add data: %s", mdb_strerror (rc)); - return FALSE; - } + if (scope == AS_COMPONENT_SCOPE_SYSTEM) + path = g_build_filename (cache_root, NULL); + else + path = g_build_filename (cache_root, "user", NULL); - return TRUE; + g_mkdir_with_parents (path, 0755); + return path; } /** - * as_cache_txn_delete: + * as_cache_components_to_internal_xml: * - * Remove key from the database. + * Convert a list of components into internal binary XML that we will store + * on disk for fast querying. + * + * Returns: Valid collection XML metadata for internal cache use. */ -inline static gboolean -lmdb_txn_delete (MDB_txn *txn, MDB_dbi dbi, const gchar *key, GError **error) +static XbSilo* +as_cache_components_to_internal_xb (AsCache *cache, + GPtrArray *cpts, + gboolean refine, + gpointer refine_func_udata, + GError **error) { - MDB_val dkey; - gint rc; - dkey = lmdb_str_to_dbval (key, -1); + AsCachePrivate *priv = GET_PRIVATE (cache); + g_autofree gchar *xml_data = NULL; + g_autoptr(XbBuilder) builder = NULL; + g_autoptr(XbBuilderSource) bsource = NULL; + g_autoptr(GError) tmp_error = NULL; + XbSilo *silo; + xmlNode *root; - rc = mdb_del (txn, dbi, &dkey, NULL); - if ((rc != MDB_NOTFOUND) && (rc != MDB_SUCCESS)) { - g_set_error (error, - AS_CACHE_ERROR, - AS_CACHE_ERROR_FAILED, - "Unable to remove data: %s", mdb_strerror (rc)); - return FALSE; + root = xmlNewNode (NULL, (xmlChar*) "components"); + for (guint i = 0; i < cpts->len; i++) { + xmlNode *node; + AsComponent *cpt = AS_COMPONENT (g_ptr_array_index (cpts, i)); + + if (refine && priv->cpt_refine_func != NULL) { + (*priv->cpt_refine_func) (cpt, TRUE, refine_func_udata); + } else { + /* ensure search token cache is generated */ + as_component_create_token_cache (cpt); + } + + /* serialize to node */ + node = as_component_to_xml_node (cpt, priv->context, NULL); + if (node == NULL) + continue; + xmlAddChild (root, node); } - return rc != MDB_NOTFOUND; -} + xml_data = as_xml_node_to_str (root, &tmp_error); + if (xml_data == NULL) { + g_propagate_prefixed_error (error, + tmp_error, + "Unable to serialize XML for caching:"); + return NULL; + } -/** - * lmdb_txn_delete_by_hash: - * - * Remove key from the database. - */ -inline static gboolean -lmdb_txn_delete_by_hash (MDB_txn *txn, MDB_dbi dbi, const guint8 *key, GError **error) -{ - MDB_val dkey; - gint rc; - dkey.mv_size = AS_CACHE_CHECKSUM_LEN; - dkey.mv_data = (void*) key; + builder = xb_builder_new (); + /* add our version to the correctness hash */ + xb_builder_append_guid (builder, PACKAGE_VERSION); - rc = mdb_del (txn, dbi, &dkey, NULL); - if ((rc != MDB_NOTFOUND) && (rc != MDB_SUCCESS)) { - g_set_error (error, - AS_CACHE_ERROR, - AS_CACHE_ERROR_FAILED, - "Unable to remove data by hash key: %s", mdb_strerror (rc)); - return FALSE; + bsource = xb_builder_source_new (); + if (!xb_builder_source_load_xml (bsource, + xml_data, + XB_BUILDER_SOURCE_FLAG_NONE, + &tmp_error)) { + g_propagate_prefixed_error (error, + tmp_error, + "Unable to load XML for compiling:"); + return NULL; } + xb_builder_import_source (builder, bsource); - return rc != MDB_NOTFOUND; + silo = xb_builder_compile (builder, + XB_BUILDER_COMPILE_FLAG_NONE, + NULL, + &tmp_error); + if (silo == NULL) { + g_propagate_prefixed_error (error, + tmp_error, + "Unable to compile binary XML for caching:"); + return NULL; + } else { + return silo; + } } /** - * as_cache_put_kv: + * as_cache_register_addons_for_component: * - * Add key/value pair to the database + * Associate available addons with this component. */ static gboolean -as_cache_put_kv (AsCache *cache, MDB_dbi dbi, const gchar *key, const gchar *value, GError **error) +as_cache_register_addons_for_component (AsCache *cache, AsComponent *cpt, GError **error) { - MDB_txn *txn = as_cache_transaction_new (cache, 0, error); - if (txn == NULL) - return FALSE; + g_autoptr(GPtrArray) addons = NULL; - if (!as_cache_txn_put_kv_str (cache, txn, dbi, key, value, error)) { - lmdb_transaction_abort (txn); + addons = as_cache_get_components_by_extends (cache, + as_component_get_id (cpt), + error); + if (addons == NULL) return FALSE; - } - return lmdb_transaction_commit (txn, error); + for (guint i = 0; i < addons->len; i++) + as_component_add_addon (cpt, g_ptr_array_index (addons, i)); + + return TRUE; } /** - * as_cache_txn_get_value: + * as_cache_component_from_node: + * + * Get component from an xmlb node. */ -static MDB_val -as_cache_txn_get_value (AsCache *cache, MDB_txn *txn, MDB_dbi dbi, const gchar *key, GError **error) +static inline AsComponent* +as_cache_component_from_node (AsCache *cache, AsCacheSection *csec, XbNode *node, GError **error) { AsCachePrivate *priv = GET_PRIVATE (cache); - MDB_cursor *cur; - MDB_val dkey; - MDB_val dval = {0, NULL}; - gsize key_len; - g_autofree gchar *key_hash = NULL; - gint rc; - - if ((key == NULL) || (key[0] == '\0')) - return dval; - - /* if key is too long, we added it as hash. So look for the hash instead. */ - key_len = sizeof(gchar) * strlen (key); - if (key_len > priv->max_keysize) { - key_hash = g_compute_checksum_for_string (AS_CACHE_CHECKSUM, key, key_len); - dkey = lmdb_str_to_dbval (key_hash, -1); - } else { - dkey = lmdb_str_to_dbval (key, key_len); - } + g_autoptr(AsComponent) cpt = NULL; + g_autofree gchar *xml_data = NULL; + xmlDoc *doc; + xmlNode *root; - rc = mdb_cursor_open (txn, dbi, &cur); - if (rc != MDB_SUCCESS) { - g_set_error (error, - AS_CACHE_ERROR, - AS_CACHE_ERROR_FAILED, - "Unable to read data (no cursor): %s", mdb_strerror (rc)); - return dval; - } + xml_data = xb_node_export (node, XB_NODE_EXPORT_FLAG_NONE, error); + if (xml_data == NULL) + return NULL; - rc = mdb_cursor_get (cur, &dkey, &dval, MDB_SET); - if (rc == MDB_NOTFOUND) { - mdb_cursor_close(cur); - return dval; + doc = as_xml_parse_document (xml_data, -1, error); + if (doc == NULL) + return NULL; + root = xmlDocGetRootElement (doc); + + cpt = as_component_new (); + if (!as_component_load_from_xml (cpt, priv->context, root, error)) { + xmlFreeDoc (doc); + return NULL; } - if (rc != MDB_SUCCESS) { - g_set_error (error, - AS_CACHE_ERROR, - AS_CACHE_ERROR_FAILED, - "Unable to read data: %s", mdb_strerror (rc)); - mdb_cursor_close(cur); - return dval; + + /* find addons (if there are any) - ensure addons don't have addons themselves */ + if ((as_component_get_kind (cpt) != AS_COMPONENT_KIND_ADDON) && + !as_cache_register_addons_for_component (cache, cpt, error)) { + xmlFreeDoc (doc); + return NULL; } - mdb_cursor_close(cur); - return dval; + /* refine for deserialization */ + if (priv->cpt_refine_func != NULL && !csec->is_mask) + (*priv->cpt_refine_func) (cpt, FALSE, csec->refine_func_udata); + + xmlFreeDoc (doc); + return g_steal_pointer (&cpt); } /** - * as_txn_get_value_by_hash: + * as_cache_remove_section_file: + * + * Safely remove a cache section file from disk. */ -static MDB_val -as_txn_get_value_by_hash (AsCache *cache, MDB_txn *txn, MDB_dbi dbi, const guint8 *hash, GError **error) +static void +as_cache_remove_section_file (AsCache *cache, AsCacheSection *csec) { - MDB_cursor *cur; - MDB_val dkey; - MDB_val dval = {0, NULL}; - gint rc; + g_autofree gchar *rnd_suffix = NULL; + g_autofree gchar *fname_old = NULL; - if (hash == NULL) - return dval; + if (!g_file_test (csec->fname, G_FILE_TEST_EXISTS)) { + /* file is already gone */ + return; + } - dkey.mv_size = AS_CACHE_CHECKSUM_LEN; - dkey.mv_data = (void*) hash; + /* random temporary name, in case multiple processes try to delete this at the same time */ + rnd_suffix = as_random_alnum_string (6); + fname_old = g_strconcat (csec->fname, rnd_suffix, ".old", NULL); - rc = mdb_cursor_open (txn, dbi, &cur); - if (rc != MDB_SUCCESS) { - g_set_error (error, - AS_CACHE_ERROR, - AS_CACHE_ERROR_FAILED, - "Unable to read data (no cursor): %s", mdb_strerror (rc)); - return dval; - } + /* just in case the file exists by any chance, we drop it */ + g_unlink (fname_old); - rc = mdb_cursor_get (cur, &dkey, &dval, MDB_SET); - if (rc == MDB_NOTFOUND) { - mdb_cursor_close(cur); - return dval; - } - if (rc != MDB_SUCCESS) { - g_set_error (error, - AS_CACHE_ERROR, - AS_CACHE_ERROR_FAILED, - "Unable to read data (using hash key): %s", mdb_strerror (rc)); - mdb_cursor_close(cur); - return dval; + if (g_rename (csec->fname, fname_old) == -1) { + /* some other process may have done this already */ + g_debug ("Unable to rename stale cache file '%s': %s", csec->fname, g_strerror (errno)); + g_unlink (csec->fname); + return; } - mdb_cursor_close(cur); - return dval; + /* we should be able to remove the file at this point */ + if (g_unlink (fname_old) == -1) + g_warning ("Unable to unlink old cache file '%s': %s", fname_old, g_strerror (errno)); } /** - * as_cache_get_value: + * as_cache_build_section_key: + * + * Generate a cache key that we can use in filenames from an + * arbitrary string that the user may have control over. + * This will also prepend any locale information. */ -inline static MDB_val -as_cache_get_value (AsCache *cache, MDB_dbi dbi, const gchar *key, GError **error) +static gchar* +as_cache_build_section_key (AsCache *cache, const gchar *str) { - MDB_txn *txn; - GError *tmp_error = NULL; - MDB_val data = {0, NULL}; - - txn = as_cache_transaction_new (cache, MDB_RDONLY, error); - if (txn == NULL) - return data; - - data = as_cache_txn_get_value (cache, txn, dbi, key, &tmp_error); - if (tmp_error != NULL) { - g_propagate_error (error, tmp_error); - lmdb_transaction_abort (txn); - return data; - } - - lmdb_transaction_commit (txn, NULL); - return data; + AsCachePrivate *priv = GET_PRIVATE (cache); + GString *gstr; + + gstr = g_string_new (str); + if (gstr->str[0] == '/') + g_string_erase (gstr, 0, 1); + g_string_prepend_c (gstr, '-'); + g_string_prepend (gstr, priv->locale); + if (gstr->str[gstr->len] == '/') + g_string_truncate (gstr, gstr->len - 1); + + as_gstring_replace2 (gstr, "/", "_", -1); + return g_string_free (gstr, FALSE); } /** - * as_cache_check_opened: + * as_cache_get_section_fname_for: */ -inline static gboolean -as_cache_check_opened (AsCache *cache, gboolean allow_floating, GError **error) +static gchar* +as_cache_get_section_fname_for (AsCache *cache, + AsCacheScope cache_scope, + AsComponentScope scope, + const gchar *section_key) { - AsCachePrivate *priv = GET_PRIVATE (cache); - g_autoptr(GMutexLocker) locker = g_mutex_locker_new (&priv->mutex); - - if (!allow_floating && priv->floating) { - g_set_error (error, - AS_CACHE_ERROR, - AS_CACHE_ERROR_FLOATING, - "Can not perform this action on a floating cache."); - return FALSE; - } + g_autofree gchar *cache_full_dir = NULL; + g_autofree gchar *cache_basename = NULL; - if (priv->opened) - return TRUE; + cache_full_dir = as_cache_get_root_dir_with_scope (cache, + cache_scope, + scope); + cache_basename = g_strconcat (section_key, ".xb", NULL); - g_set_error (error, - AS_CACHE_ERROR, - AS_CACHE_ERROR_NOT_OPEN, - "Can not perform this action on an unopened cache."); - return FALSE; + return g_build_filename (cache_full_dir, cache_basename, NULL); } /** - * as_cache_component_to_xml: + * as_cache_set_contents_internal: */ -static gchar* -as_cache_component_to_xml (AsCache *cache, AsComponent *cpt) +static gboolean +as_cache_set_contents_internal (AsCache *cache, + AsComponentScope scope, + AsFormatStyle source_format_style, + gboolean is_os_data, + GPtrArray *cpts, + const gchar *cache_key, + gpointer refine_user_data, + GError **error) { AsCachePrivate *priv = GET_PRIVATE (cache); - xmlDoc *doc; - xmlNode *node; - xmlBufferPtr buf; - xmlSaveCtxtPtr sctx; - gchar *res; + g_autofree gchar *internal_section_key = NULL; + g_autofree gchar *section_key = NULL; + g_autofree gchar *cache_full_dir = NULL; + g_autoptr(AsCacheSection) csec = NULL; + g_autoptr(GFile) file = NULL; + g_autoptr(GError) tmp_error = NULL; + g_autoptr(GMutexLocker) locker = NULL; + gboolean ret = TRUE; + + section_key = as_cache_build_section_key (cache, cache_key); + internal_section_key = g_strconcat (as_component_scope_to_string (scope), ":", section_key, NULL); + g_debug ("Storing cache data for section: %s", internal_section_key); + locker = g_mutex_locker_new (&priv->sec_mutex); + + /* ensure we can write to the cache location */ + cache_full_dir = as_cache_get_root_dir_with_scope (cache, + AS_CACHE_SCOPE_WRITABLE, /* only the "user" scope is ever writable */ + scope); + if (!as_utils_is_writable (cache_full_dir)) { + g_set_error (error, + AS_CACHE_ERROR, + AS_CACHE_ERROR_PERMISSIONS, + _("Cache location '%s' is not writable."), cache_full_dir); + return FALSE; + } - node = as_component_to_xml_node (cpt, priv->context, NULL); - if (node == NULL) - return NULL; + /* replace old section, in case one with the same key exists */ + for (guint i = 0; i < priv->sections->len; i++) { + AsCacheSection *csec_entry = g_ptr_array_index (priv->sections, i); + if (g_strcmp0 (csec_entry->key, internal_section_key) == 0) { + as_cache_remove_section_file (cache, csec_entry); + g_ptr_array_remove_index_fast (priv->sections, i); + break; + } + } - doc = xmlNewDoc ((xmlChar*) NULL); - xmlDocSetRootElement (doc, node); + /* create new section */ + csec = as_cache_section_new (internal_section_key); + csec->is_os_data = is_os_data && scope == AS_COMPONENT_SCOPE_SYSTEM; + csec->scope = scope; + csec->format_style = source_format_style; + csec->fname = as_cache_get_section_fname_for (cache, AS_CACHE_SCOPE_WRITABLE, scope, section_key); + csec->refine_func_udata = refine_user_data; + + csec->silo = as_cache_components_to_internal_xb (cache, + cpts, + TRUE, /* refine */ + csec->refine_func_udata, + error); + if (csec->silo == NULL) + return FALSE; - buf = xmlBufferCreate (); - sctx = xmlSaveToBuffer (buf, "utf-8", XML_SAVE_NO_DECL); - xmlSaveDoc (sctx, doc); - xmlSaveClose (sctx); + /* write data to cache directory */ + g_debug ("Writing cache file: %s", csec->fname); + file = g_file_new_for_path (csec->fname); + if (!xb_silo_save_to_file (csec->silo, file, NULL, &tmp_error)) { + g_propagate_prefixed_error (error, + tmp_error, + "Unable to write cache file:"); + ret = FALSE; + } - res = g_strdup ((const gchar*) xmlBufferContent (buf)); - xmlBufferFree (buf); - xmlFreeDoc (doc); - - return res; -} - -/** - * as_cache_open: - * @cache: An instance of #AsCache. - * @fname: The filename of the cache file (or ":memory" or ":temporary" for an in-memory or temporary cache) - * @locale: The locale with which the cache should be created, if a new cache was created. - * @error: A #GError or %NULL. - * - * Open/create an AppStream cache file. - * - * Returns: %TRUE if cache was opened successfully. - **/ -gboolean -as_cache_open (AsCache *cache, const gchar *fname, const gchar *locale, GError **error) -{ - AsCachePrivate *priv = GET_PRIVATE (cache); - GError *tmp_error = NULL; - gint rc; - MDB_txn *txn = NULL; - MDB_dbi db_config; - g_autofree gchar *cache_format = NULL; - unsigned int db_flags; - gboolean nosync; - gboolean readonly; - g_autoptr(GMutexLocker) locker = NULL; - int db_fd; - g_autofree gchar *volatile_fname = NULL; - - /* close cache in case it was open */ - as_cache_close (cache); - - locker = g_mutex_locker_new (&priv->mutex); - - rc = mdb_env_create (&priv->db_env); - if (rc != MDB_SUCCESS) { - g_set_error (error, - AS_CACHE_ERROR, - AS_CACHE_ERROR_FAILED, - "Unable to initialize an MDB environment: %s", mdb_strerror (rc)); - goto fail; - } - - /* we currently have 9 DBs */ - rc = mdb_env_set_maxdbs (priv->db_env, 9); - if (rc != MDB_SUCCESS) { - g_set_error (error, - AS_CACHE_ERROR, - AS_CACHE_ERROR_FAILED, - "Unable to set MDB max DB count: %s", mdb_strerror (rc)); - goto fail; - } - - rc = mdb_env_set_mapsize (priv->db_env, LMDB_DB_SIZE); - if (rc != MDB_SUCCESS) { - g_set_error (error, - AS_CACHE_ERROR, - AS_CACHE_ERROR_FAILED, - "Unable to set DB mapsize: %s", mdb_strerror (rc)); - goto fail; - } - - /* determine database mode */ - db_flags = MDB_NOSUBDIR | MDB_NOMETASYNC | MDB_NOLOCK; - nosync = priv->nosync; - readonly = priv->readonly; - - /* determine where to put a volatile database */ - if (g_strcmp0 (fname, ":temporary") == 0) { - g_autofree gchar *volatile_dir = as_get_user_cache_dir (); - volatile_fname = g_build_filename (volatile_dir, "appcache-XXXXXX.mdb", NULL); - - if (g_mkdir_with_parents (volatile_dir, 0755) != 0) { - g_set_error (error, - AS_CACHE_ERROR, - AS_CACHE_ERROR_FAILED, - "Unable to create cache directory (%s)",volatile_dir); - goto fail; - } - - /* we can skip syncs in temporary mode - in case of a crash, the data is lost anyway */ - nosync = TRUE; - - /* readonly mode makes no sense if we have a temporary cache */ - readonly = FALSE; - - } else if (g_strcmp0 (fname, ":memory") == 0) { - volatile_fname = g_build_filename (g_get_user_runtime_dir (), "appstream-cache-XXXXXX.mdb", NULL); - /* we can skip syncs in in-memory mode - they don't mean anything anyway*/ - nosync = TRUE; - - /* readonly mode makes no sense if we have an in-memory cache */ - readonly = FALSE; - } - - if (readonly) - db_flags |= MDB_RDONLY; - if (nosync) - db_flags |= MDB_NOSYNC; - - if (volatile_fname != NULL) { - gint fd; - - g_free (priv->fname); - priv->fname = g_steal_pointer (&volatile_fname); - fd = g_mkstemp (priv->fname); - if (fd < 0) { - g_set_error (error, - AS_CACHE_ERROR, - AS_CACHE_ERROR_FAILED, - "Unable to open temporary cache file: %s", g_strerror (errno)); - goto fail; - } else { - close (fd); - } - - g_free (priv->volatile_db_fname); - priv->volatile_db_fname = g_strdup (priv->fname); - } else { - g_free (priv->fname); - priv->fname = g_strdup (fname); - } - - if (readonly) - g_debug ("Opening cache file for reading only: %s", priv->fname); - else - g_debug ("Opening cache file: %s", priv->fname); - rc = mdb_env_open (priv->db_env, - priv->fname, - db_flags, - 0755); - if (rc != MDB_SUCCESS) { - g_set_error (error, - AS_CACHE_ERROR, - AS_CACHE_ERROR_FAILED, - "Unable to open cache: %s", mdb_strerror (rc)); - goto fail; - } - - /* set FD_CLOEXEC manually. LMDB should do that, but it doesn't: - https://www.openldap.org/lists/openldap-bugs/201702/msg00003.html */ - if (mdb_env_get_fd (priv->db_env, &db_fd) == MDB_SUCCESS) { - int db_fd_flags = fcntl (db_fd, F_GETFD); - if (db_fd_flags != -1) - fcntl (db_fd, F_SETFD, db_fd_flags | FD_CLOEXEC); - } - - /* unlink the file, so it gets removed as soon as we don't need it anymore */ - if (priv->volatile_db_fname != NULL) - g_unlink (priv->volatile_db_fname); - - /* retrieve the maximum keysize allowed for this database */ - priv->max_keysize = mdb_env_get_maxkeysize (priv->db_env); - - txn = as_cache_transaction_new (cache, - readonly? MDB_RDONLY : 0, - &tmp_error); - if (txn == NULL) { - g_propagate_error (error, tmp_error); - goto fail; - } - - /* cache settings */ - if (!as_cache_open_subdb (txn, "config", &db_config, &tmp_error)) { - g_propagate_error (error, tmp_error); - goto fail; - } - - /* component data as XML */ - if (!as_cache_open_subdb (txn, "components", &priv->db_cpts, &tmp_error)) { - g_propagate_error (error, tmp_error); - goto fail; - } - - /* component-ID mapping */ - if (!as_cache_open_subdb (txn, "cpt_ids", &priv->db_cids, &tmp_error)) { - g_propagate_error (error, tmp_error); - goto fail; - } - - /* full-text search index */ - if (!as_cache_open_subdb (txn, "fts", &priv->db_fts, &tmp_error)) { - g_propagate_error (error, tmp_error); - goto fail; - } - - /* categories */ - if (!as_cache_open_subdb (txn, "categories", &priv->db_cats, &tmp_error)) { - g_propagate_error (error, tmp_error); - goto fail; - } - - /* launchables */ - if (!as_cache_open_subdb (txn, "launchables", &priv->db_launchables, &tmp_error)) { - g_propagate_error (error, tmp_error); - goto fail; - } - - /* provides */ - if (!as_cache_open_subdb (txn, "provides", &priv->db_provides, &tmp_error)) { - g_propagate_error (error, tmp_error); - goto fail; - } - - /* kinds */ - if (!as_cache_open_subdb (txn, "kinds", &priv->db_kinds, &tmp_error)) { - g_propagate_error (error, tmp_error); - goto fail; - } - - /* addons */ - if (!as_cache_open_subdb (txn, "addons", &priv->db_addons, &tmp_error)) { - g_propagate_error (error, tmp_error); - goto fail; - } - - if (!lmdb_transaction_commit (txn, &tmp_error)) { - g_propagate_error (error, tmp_error); - goto fail; - } - - cache_format = lmdb_val_strdup (as_cache_get_value (cache, db_config, "format", &tmp_error)); - if (tmp_error != NULL) { - g_propagate_error (error, tmp_error); - goto fail; - } - if (cache_format == NULL) { - /* the value was empty, we most likely have a new cache file */ - if (readonly) { - g_set_error (error, - AS_CACHE_ERROR, - AS_CACHE_ERROR_WRONG_FORMAT, - "Cache format missing on read-only cache."); - goto fail; - } else { - if (!as_cache_put_kv (cache, db_config, "format", CACHE_FORMAT_VERSION, error)) - goto fail; - } - } else if (g_strcmp0 (cache_format, CACHE_FORMAT_VERSION) != 0) { - /* we try to load an incompatible cache version - emit an error, so it can be recreated */ - g_set_error (error, - AS_CACHE_ERROR, - AS_CACHE_ERROR_WRONG_FORMAT, - "The cache format version is unsupported."); - goto fail; - } - - g_free (priv->locale); - priv->locale = lmdb_val_strdup (as_cache_get_value (cache, db_config, "locale", &tmp_error)); - if (tmp_error != NULL) { - g_propagate_error (error, tmp_error); - goto fail; - } - if (priv->locale == NULL) { - /* the value was empty, we most likely have a new cache file - so set a locale for it if we can */ - if (readonly) { - g_set_error (error, - AS_CACHE_ERROR, - AS_CACHE_ERROR_LOCALE_MISMATCH, - "Locale value missing on read-only cache."); - goto fail; - } else { - if (!as_cache_put_kv (cache, db_config, "locale", locale, error)) - goto fail; - } - priv->locale = g_strdup (locale); - } else if (g_strcmp0 (priv->locale, locale) != 0) { - /* we expected a different locale than this cache has - throw an error, just to be safe */ - g_set_error (error, - AS_CACHE_ERROR, - AS_CACHE_ERROR_LOCALE_MISMATCH, - "We expected locale '%s', but the cache was build for '%s'.", locale, priv->locale); - goto fail; - } - - /* set locale for XML serialization */ - as_context_set_locale (priv->context, priv->locale); - - priv->opened = TRUE; - return TRUE; -fail: - lmdb_transaction_abort (txn); - if (priv->db_env != NULL) - mdb_env_close (priv->db_env); - return FALSE; -} - -/** - * as_cache_open2: - * @cache: An instance of #AsCache. - * @locale: The locale with which the cache should be created, if a new cache was created. - * @error: A #GError or %NULL. - * - * Open/create an AppStream cache file based on the previously set filename. - * (This is equivalent to reopening the previous database - to create a new temporary cache - * and discard the old one, use open()) - * - * Returns: %TRUE if cache was opened successfully. - **/ -gboolean -as_cache_open2 (AsCache *cache, const gchar *locale, GError **error) -{ - AsCachePrivate *priv = GET_PRIVATE (cache); - g_autofree gchar *fname = NULL; - - if (priv->fname == NULL) { - g_set_error (error, - AS_CACHE_ERROR, - AS_CACHE_ERROR_NO_FILENAME, - "No location was set for this cache."); - return FALSE; - } - - /* we need to duplicate this here, as open() frees priv->fname in case - * it contains a placeholder like :temporary */ - fname = g_strdup (priv->fname); - - return as_cache_open (cache, fname, locale, error); -} - -/** - * as_cache_close: - * @cache: An instance of #AsCache. - * - * Close an AppStream cache file. - * This function can be called after the cache has been opened to - * explicitly close it and reuse the #AsCache instance. - * It will also be called when the object is freed. - * - * Returns: %TRUE if cache was opened successfully. - **/ -gboolean -as_cache_close (AsCache *cache) -{ - AsCachePrivate *priv = GET_PRIVATE (cache); - g_autoptr(GMutexLocker) locker = g_mutex_locker_new (&priv->mutex); - - if (!priv->opened) - return FALSE; - - if (!priv->readonly) - mdb_env_sync (priv->db_env, 1); - mdb_env_close (priv->db_env); - - /* ensure any temporary file is gone, in case we used an in-memory database */ - if (priv->volatile_db_fname != NULL) { - g_remove (priv->volatile_db_fname); - g_free (priv->volatile_db_fname); - priv->volatile_db_fname = NULL; - } - - priv->opened = FALSE; - return TRUE; -} - -/** - * as_cache_hash_set_append: - * - * Helper function for as_cache_insert() - */ -static gboolean -as_cache_hash_set_append (guint8 **list, gsize *list_len, const guint8 *new_hash) -{ - g_assert_cmpint (*list_len % AS_CACHE_CHECKSUM_LEN, ==, 0); - - /* check if the list already contains the hash that should be added */ - for (gsize i = 0; i < *list_len; i += AS_CACHE_CHECKSUM_LEN) { - if (memcmp ((*list) + i, new_hash, AS_CACHE_CHECKSUM_LEN) == 0) - return FALSE; /* hash was already in the set */ - } - - /* add new hash to the list */ - *list_len = *list_len + AS_CACHE_CHECKSUM_LEN; - *list = g_realloc_n (*list, *list_len, sizeof(guint8)); - - memcpy ((*list) + (*list_len) - AS_CACHE_CHECKSUM_LEN, new_hash, AS_CACHE_CHECKSUM_LEN); - return TRUE; -} - -/** - * as_cache_hash_match_dict_insert: - * - * Helper function for as_cache_insert() FTS - */ -static gboolean -as_cache_hash_match_dict_insert (guint8 **dict, gsize *dict_len, const guint8 *hash, AsTokenType match_val) -{ - const gsize ENTRY_LEN = sizeof(AsTokenType) + AS_CACHE_CHECKSUM_LEN * sizeof(guint8); - gsize insert_idx = *dict_len; - g_assert_cmpint (*dict_len % ENTRY_LEN, ==, 0); - - /* check if the list already contains the hash that should be added */ - for (gsize i = 0; i < *dict_len; i += ENTRY_LEN) { - if (memcmp ((*dict) + i, hash, AS_CACHE_CHECKSUM_LEN) == 0) { - if (((AsTokenType) *(*dict + i + AS_CACHE_CHECKSUM_LEN)) == match_val) - return FALSE; /* the entry is already known, we will add nothing to the dict */ - insert_idx = i; /* change index so we override the existing entry */ - break; - } - } - - if (insert_idx >= *dict_len) { - /* the entry is new, so we need to add space in the list for it */ - - *dict_len = *dict_len + ENTRY_LEN; - *dict = g_realloc (*dict, *dict_len); - } - - memcpy ((*dict) + insert_idx, hash, AS_CACHE_CHECKSUM_LEN); - memcpy ((*dict) + insert_idx + AS_CACHE_CHECKSUM_LEN, &match_val, sizeof(AsTokenType)); - - return TRUE; -} - -/** - * as_cache_insert: - * @cache: An instance of #AsCache. - * @cpt: The #AsComponent to insert. - * @error: A #GError or %NULL. - * - * Insert a new component into the cache. - * - * Returns: %TRUE if there was no error. - **/ -gboolean -as_cache_insert (AsCache *cache, AsComponent *cpt, GError **error) -{ - AsCachePrivate *priv = GET_PRIVATE (cache); - MDB_txn *txn = NULL; - g_autofree gchar *xml_data = NULL; - g_autofree guint8 *cpt_checksum = NULL; - GError *tmp_error = NULL; - - GHashTable *token_cache; - GHashTableIter tc_iter; - gpointer tc_key, tc_value; - - GPtrArray *categories; - GPtrArray *launchables; - GPtrArray *provides; - GPtrArray *extends; - - g_autoptr(GMutexLocker) locker = NULL; - - if (!as_cache_check_opened (cache, TRUE, error)) - return FALSE; - locker = g_mutex_locker_new (&priv->mutex); - - if (priv->floating) { - /* floating cache, don't really add this component yet but stage it in the internal map */ - g_hash_table_insert (priv->cpt_map, - g_strdup (as_component_get_data_id (cpt)), - g_object_ref (cpt)); - g_hash_table_add (priv->cid_set, - g_strdup (as_component_get_id (cpt))); - return TRUE; - } - - txn = as_cache_transaction_new (cache, 0, error); - if (txn == NULL) - return FALSE; - - /* add a "fake" launchable entry for desktop-apps that failed to include one. - * This is used for legacy compatibility */ - if ((as_component_get_kind (cpt) == AS_COMPONENT_KIND_DESKTOP_APP) && - (as_component_get_launchables (cpt)->len <= 0)) { - if (g_str_has_suffix (as_component_get_id (cpt), ".desktop")) { - g_autoptr(AsLaunchable) launchable = as_launchable_new (); - as_launchable_set_kind (launchable, AS_LAUNCHABLE_KIND_DESKTOP_ID); - as_launchable_add_entry (launchable, as_component_get_id (cpt)); - as_component_add_launchable (cpt, launchable); - } - } - - /* add the component data itself to the cache */ - xml_data = as_cache_component_to_xml (cache, cpt); - as_generate_cache_checksum (as_component_get_data_id (cpt), - -1, - &cpt_checksum, - NULL); - - if (!as_txn_put_hash_kv_str (txn, - priv->db_cpts, - cpt_checksum, - xml_data, - error)) { - goto fail; - } - - /* populate full-text search index */ - as_component_create_token_cache (cpt); - token_cache = as_component_get_token_cache_table (cpt); - - g_hash_table_iter_init (&tc_iter, token_cache); - while (g_hash_table_iter_next (&tc_iter, &tc_key, &tc_value)) { - MDB_val fts_val; - g_autofree guint8 *match_list = NULL; - gsize match_list_len; - const gchar *token_str; - size_t token_len; - AsTokenType *match_pval; - - /* we ignore tokens which are too long to be database keys */ - token_str = (const gchar*) tc_key; - token_len = sizeof(gchar) * strlen (token_str); - if (token_len == 0) { - g_warning ("Ignored empty search token for component '%s'", as_component_get_data_id (cpt)); - continue; - } - if (token_len > priv->max_keysize) { - g_debug ("Ignored search token '%s': Too long to be stored in the cache.", token_str); - continue; - } - - /* get existing fts match value - we deliberately ignore any errors while reading here */ - fts_val = as_cache_txn_get_value (cache, - txn, - priv->db_fts, - token_str, - NULL); - - match_pval = (AsTokenType *) tc_value; - /* TODO: There is potential to save on allocations here */ -#if GLIB_CHECK_VERSION(2,68,0) - match_list = g_memdup2 (fts_val.mv_data, fts_val.mv_size); -#else - match_list = g_memdup (fts_val.mv_data, fts_val.mv_size); -#endif - match_list_len = fts_val.mv_size; - - if (as_cache_hash_match_dict_insert (&match_list, &match_list_len, cpt_checksum, *match_pval)) { - /* our checksum was not in the list yet, so add it */ - if (!as_cache_txn_put_kv_bytes (cache, - txn, - priv->db_fts, - token_str, - match_list, - match_list_len, - error)) { - goto fail; - } - } - } - - /* register this component with the CID mapping */ - { - g_autofree guint8 *hash_list = NULL; - gsize hash_list_len; - hash_list = lmdb_val_memdup (as_cache_txn_get_value (cache, - txn, - priv->db_cids, - as_component_get_id (cpt), - &tmp_error), &hash_list_len); - if (tmp_error != NULL) { - g_propagate_error (error, tmp_error); - goto fail; - } - - if (as_cache_hash_set_append (&hash_list, &hash_list_len, cpt_checksum)) { - /* our checksum was not in the list yet, so add it */ - if (!as_cache_txn_put_kv_bytes (cache, - txn, - priv->db_cids, - as_component_get_id (cpt), - hash_list, - hash_list_len, - error)) { - goto fail; - } - } - } - - /* add category mapping */ - categories = as_component_get_categories (cpt); - for (guint i = 0; i < categories->len; i++) { - g_autofree guint8 *hash_list = NULL; - gsize hash_list_len; - const gchar *category = (const gchar*) g_ptr_array_index (categories, i); - - hash_list = lmdb_val_memdup (as_cache_txn_get_value (cache, - txn, - priv->db_cats, - category, - &tmp_error), &hash_list_len); - if (tmp_error != NULL) { - g_propagate_error (error, tmp_error); - goto fail; - } - - if (as_cache_hash_set_append (&hash_list, &hash_list_len, cpt_checksum)) { - /* this component was not yet registered with the category */ - if (!as_cache_txn_put_kv_bytes (cache, - txn, - priv->db_cats, - category, - hash_list, - hash_list_len, - error)) { - goto fail; - } - } - } - - /* add launchables mapping */ - launchables = as_component_get_launchables (cpt); - for (guint i = 0; i < launchables->len; i++) { - GPtrArray *entries; - AsLaunchable *launchable = AS_LAUNCHABLE (g_ptr_array_index (launchables, i)); - - entries = as_launchable_get_entries (launchable); - for (guint j = 0; j < entries->len; j++) { - g_autofree gchar *entry_key = NULL; - g_autofree guint8 *hash_list = NULL; - gsize hash_list_len; - const gchar *entry = (const gchar*) g_ptr_array_index (entries, j); - - entry_key = g_strconcat (as_launchable_kind_to_string (as_launchable_get_kind (launchable)), entry, NULL); - hash_list = lmdb_val_memdup (as_cache_txn_get_value (cache, - txn, - priv->db_launchables, - entry_key, - &tmp_error), &hash_list_len); - if (tmp_error != NULL) { - g_propagate_error (error, tmp_error); - goto fail; - } - - if (as_cache_hash_set_append (&hash_list, &hash_list_len, cpt_checksum)) { - /* add component checksum to launchable list */ - if (!as_cache_txn_put_kv_bytes (cache, - txn, - priv->db_launchables, - entry_key, - hash_list, - hash_list_len, - error)) { - goto fail; - } - } - } - } - - /* add provides mapping */ - provides = as_component_get_provided (cpt); - for (guint i = 0; i < provides->len; i++) { - GPtrArray *items; - AsProvided *prov = AS_PROVIDED (g_ptr_array_index (provides, i)); - - items = as_provided_get_items (prov); - for (guint j = 0; j < items->len; j++) { - g_autofree gchar *item_key = NULL; - g_autofree guint8 *hash_list = NULL; - gsize hash_list_len; - const gchar *item = (const gchar*) g_ptr_array_index (items, j); - - item_key = g_strconcat (as_provided_kind_to_string (as_provided_get_kind (prov)), item, NULL); - hash_list = lmdb_val_memdup (as_cache_txn_get_value (cache, - txn, - priv->db_provides, - item_key, - &tmp_error), &hash_list_len); - if (tmp_error != NULL) { - g_propagate_error (error, tmp_error); - goto fail; - } - - if (as_cache_hash_set_append (&hash_list, &hash_list_len, cpt_checksum)) { - /* add component checksum to launchable list */ - if (!as_cache_txn_put_kv_bytes (cache, - txn, - priv->db_provides, - item_key, - hash_list, - hash_list_len, - error)) { - goto fail; - } - } - } - } - - /* add kinds mapping */ - { - const gchar *kind_str = as_component_kind_to_string (as_component_get_kind (cpt)); - g_autofree guint8 *hash_list = NULL; - gsize hash_list_len; - - hash_list = lmdb_val_memdup (as_cache_txn_get_value (cache, - txn, - priv->db_kinds, - kind_str, - &tmp_error), &hash_list_len); - if (tmp_error != NULL) { - g_propagate_error (error, tmp_error); - goto fail; - } - - if (as_cache_hash_set_append (&hash_list, &hash_list_len, cpt_checksum)) { - /* add component checksum to component-kind mapping */ - if (!as_cache_txn_put_kv_bytes (cache, - txn, - priv->db_kinds, - kind_str, - hash_list, - hash_list_len, - error)) { - goto fail; - } - } - } - - /* add addon/extension mapping for quick lookup of addons */ - extends = as_component_get_extends (cpt); - if ((as_component_get_kind (cpt) == AS_COMPONENT_KIND_ADDON) && - (extends != NULL) && - (extends->len > 0)) { - for (guint i = 0; i < extends->len; i++) { - g_autofree gchar *extended_cdid = NULL; - g_autofree guint8 *hash_list = NULL; - gsize hash_list_len; - const gchar *extended_cid = (const gchar*) g_ptr_array_index (extends, i); - - extended_cdid = as_utils_build_data_id (as_component_get_scope (cpt), - as_utils_get_component_bundle_kind (cpt), - as_component_get_origin (cpt), - extended_cid, - NULL); - - hash_list = lmdb_val_memdup (as_cache_txn_get_value (cache, - txn, - priv->db_addons, - extended_cdid, - &tmp_error), &hash_list_len); - if (tmp_error != NULL) { - g_propagate_error (error, tmp_error); - goto fail; - } - - if (as_cache_hash_set_append (&hash_list, &hash_list_len, cpt_checksum)) { - /* our checksum was not in the addon list yet, so add it */ - if (!as_cache_txn_put_kv_bytes (cache, - txn, - priv->db_addons, - extended_cdid, - hash_list, - hash_list_len, - error)) { - goto fail; - } - } - } - } - - return lmdb_transaction_commit (txn, error); -fail: - lmdb_transaction_abort (txn); - return FALSE; -} - -/** - * as_cache_remove_by_data_id: - * @cache: An instance of #AsCache. - * @cdid: Component data ID. - * @error: A #GError or %NULL. - * - * Remove a component from the cache. - * - * Returns: %TRUE if component was removed. - **/ -gboolean -as_cache_remove_by_data_id (AsCache *cache, const gchar *cdid, GError **error) -{ - AsCachePrivate *priv = GET_PRIVATE (cache); - MDB_txn *txn = NULL; - g_autofree guint8 *cpt_checksum = NULL; - GError *tmp_error = NULL; - gboolean ret; - g_autoptr(GMutexLocker) locker = NULL; - - if (!as_cache_check_opened (cache, TRUE, error)) - return FALSE; - locker = g_mutex_locker_new (&priv->mutex); - - if (priv->floating) { - /* floating cache, remove only from the internal map */ - return g_hash_table_remove (priv->cpt_map, cdid); - } - - if (priv->readonly) { - as_generate_cache_checksum (cdid, -1, &cpt_checksum, NULL); - g_hash_table_add (priv->ro_removed_set, - g_steal_pointer (&cpt_checksum)); - return TRUE; - } - - txn = as_cache_transaction_new (cache, 0, error); - if (txn == NULL) - return FALSE; - - /* remove component itself from the cache */ - as_generate_cache_checksum (cdid, -1, &cpt_checksum, NULL); - - ret = lmdb_txn_delete_by_hash (txn, priv->db_cpts, cpt_checksum, &tmp_error); - if (tmp_error != NULL) { - g_propagate_error (error, tmp_error); - goto fail; - } - - if (ret) { - /* TODO: We could remove all references to the removed component from the cache here. - * Currently those are not an issue though, so we may keep them (as removing references is expensive) */ - } - - lmdb_transaction_commit (txn, &tmp_error); - if (tmp_error != NULL) { - g_propagate_error (error, tmp_error); - goto fail; - } - - return ret; -fail: - lmdb_transaction_abort (txn); - return FALSE; -} - -/** - * as_cache_component_from_dval: - * - * Get component from database entry. - */ -static inline AsComponent* -as_cache_component_from_dval (AsCache *cache, MDB_txn *txn, MDB_val dval, GError **error) -{ - AsCachePrivate *priv = GET_PRIVATE (cache); - g_autoptr(AsComponent) cpt = NULL; - xmlDoc *doc; - xmlNode *root; - - if (dval.mv_size <= 0) - return NULL; - - doc = as_xml_parse_document (dval.mv_data, dval.mv_size, error); - if (doc == NULL) - return NULL; - root = xmlDocGetRootElement (doc); - - cpt = as_component_new (); - if (!as_component_load_from_xml (cpt, priv->context, root, error)) { - xmlFreeDoc (doc); - return NULL; - } - - /* find addons (if there are any) - ensure addons don't have addons themselves */ - if ((as_component_get_kind (cpt) != AS_COMPONENT_KIND_ADDON) && - !as_cache_register_addons_for_component (cache, txn, cpt, error)) { - xmlFreeDoc (doc); - return NULL; - } - - if (priv->cpt_refine_func != NULL) - (*priv->cpt_refine_func) (cpt, priv->cpt_refine_func_udata); - - xmlFreeDoc (doc); - return g_steal_pointer (&cpt); -} - -/** - * as_cache_component_by_hash: - * - * Retrieve a component using its internal hash. - */ -static AsComponent* -as_cache_component_by_hash (AsCache *cache, MDB_txn *txn, const guint8 *cpt_hash, GError **error) -{ - AsCachePrivate *priv = GET_PRIVATE (cache); - MDB_val dval; - g_autoptr(AsComponent) cpt = NULL; - GError *tmp_error = NULL; - - /* in case we "removed" the component on a readonly cache */ - if (g_hash_table_contains (priv->ro_removed_set, cpt_hash)) - return NULL; - - dval = as_txn_get_value_by_hash (cache, txn, priv->db_cpts, cpt_hash, &tmp_error); - if (tmp_error != NULL) { - g_propagate_error (error, tmp_error); - return NULL; - } - if (dval.mv_size <= 0) - return NULL; - - cpt = as_cache_component_from_dval (cache, txn, dval, error); - if (cpt == NULL) - return NULL; /* error */ - - return g_steal_pointer (&cpt); -} - -/** - * as_cache_components_by_hash_list: - * - * Retrieve components using an internal hash list. - */ -static GPtrArray* -as_cache_components_by_hash_list (AsCache *cache, MDB_txn *txn, const guint8 *hlist, gsize hlist_len, GError **error) -{ - g_autoptr(GPtrArray) result = NULL; - GError *tmp_error = NULL; - g_assert_cmpint (hlist_len % AS_CACHE_CHECKSUM_LEN, ==, 0); - - result = g_ptr_array_new_with_free_func (g_object_unref); - if (hlist == NULL) - goto out; - - for (gsize i = 0; i < hlist_len; i += AS_CACHE_CHECKSUM_LEN) { - AsComponent *cpt; - - cpt = as_cache_component_by_hash (cache, txn, hlist + i, &tmp_error); - if (tmp_error != NULL) { - g_propagate_prefixed_error (error, tmp_error, "Failed to retrieve component data: "); - return NULL; - } - if (cpt != NULL) - g_ptr_array_add (result, cpt); - } - -out: - return g_steal_pointer (&result); -} - -/** - * as_cache_register_addons_for_component: - * - * Associate available addons with this component. - */ -static gboolean -as_cache_register_addons_for_component (AsCache *cache, MDB_txn *txn, AsComponent *cpt, GError **error) -{ - AsCachePrivate *priv = GET_PRIVATE (cache); - MDB_val dval; - g_autofree guint8 *cpt_checksum = NULL; - GError *tmp_error = NULL; - - dval = as_cache_txn_get_value (cache, - txn, - priv->db_addons, - as_component_get_data_id (cpt), - &tmp_error); - if (tmp_error != NULL) { - g_propagate_error (error, tmp_error); - return FALSE; - } - if (dval.mv_size == 0) - return TRUE; - - /* retrieve cache checksum of this component */ - as_generate_cache_checksum (as_component_get_data_id (cpt), - -1, - &cpt_checksum, - NULL); - - g_assert_cmpint (dval.mv_size % AS_CACHE_CHECKSUM_LEN, ==, 0); - for (gsize i = 0; i < dval.mv_size; i += AS_CACHE_CHECKSUM_LEN) { - const guint8 *chash = dval.mv_data + i; - g_autoptr(AsComponent) addon = NULL; - - /* ignore addon that extends itself to prevent infinite recursion */ - if (memcmp (chash, cpt_checksum, AS_CACHE_CHECKSUM_LEN) == 0) - continue; - - addon = as_cache_component_by_hash (cache, txn, chash, &tmp_error); - if (tmp_error != NULL) { - g_propagate_prefixed_error (error, tmp_error, "Failed to retrieve addon component data: "); - return FALSE; - } - - /* ensure we only link addons to the component directly, to prevent loops in refcounting and annoying - * dependency graph issues. - * Frontends can get their non-addon extensions via the "extends" property of #AsComponent and a pool query instead */ - if ((addon != NULL) && (as_component_get_kind (addon) == AS_COMPONENT_KIND_ADDON)) - as_component_add_addon (cpt, addon); - } - - return TRUE; -} - -/** - * as_cache_get_components_all: - * @cache: An instance of #AsCache. - * @error: A #GError or %NULL. - * - * Retrieve all components this cache contains. - * - * Returns: (transfer full): An array of #AsComponent - */ -GPtrArray* -as_cache_get_components_all (AsCache *cache, GError **error) -{ - AsCachePrivate *priv = GET_PRIVATE (cache); - MDB_txn *txn; - MDB_cursor *cur; - gint rc; - MDB_val dval; - MDB_val dkey; - g_autoptr(GPtrArray) results = NULL; - g_autoptr(GMutexLocker) locker = NULL; - - if (!as_cache_check_opened (cache, FALSE, error)) - return NULL; - locker = g_mutex_locker_new (&priv->mutex); - - txn = as_cache_transaction_new (cache, MDB_RDONLY, error); - if (txn == NULL) - return NULL; - - results = g_ptr_array_new_with_free_func (g_object_unref); - - rc = mdb_cursor_open (txn, priv->db_cpts, &cur); - if (rc != MDB_SUCCESS) { - g_set_error (error, - AS_CACHE_ERROR, - AS_CACHE_ERROR_FAILED, - "Unable to iterate cache (no cursor): %s", mdb_strerror (rc)); - lmdb_transaction_abort (txn); - return NULL; - } - - rc = mdb_cursor_get (cur, &dkey, &dval, MDB_FIRST); - while (rc == 0) { - AsComponent *cpt; - if (dval.mv_size <= 0) { - rc = mdb_cursor_get (cur, NULL, &dval, MDB_NEXT); - continue; - } - - /* in case we "removed" the component on a readonly cache */ - if (priv->readonly) { - g_autofree gchar *cpt_hash = g_strndup (dkey.mv_data, dkey.mv_size); - if (g_hash_table_contains (priv->ro_removed_set, cpt_hash)) - return NULL; - } - - cpt = as_cache_component_from_dval (cache, txn, dval, error); - if (cpt == NULL) - return NULL; /* error */ - g_ptr_array_add (results, cpt); - - rc = mdb_cursor_get (cur, NULL, &dval, MDB_NEXT); - } - mdb_cursor_close (cur); - - lmdb_transaction_commit (txn, NULL); - return g_steal_pointer (&results); -} - -/** - * as_cache_get_components_by_id: - * @cache: An instance of #AsCache. - * @id: The component ID to search for. - * @error: A #GError or %NULL. - * - * Retrieve components with the selected ID. - * - * Returns: (transfer full): An array of #AsComponent - */ -GPtrArray* -as_cache_get_components_by_id (AsCache *cache, const gchar *id, GError **error) -{ - AsCachePrivate *priv = GET_PRIVATE (cache); - MDB_txn *txn; - GError *tmp_error = NULL; - MDB_val dval; - GPtrArray *result = NULL; - g_autoptr(GMutexLocker) locker = NULL; - - if (!as_cache_check_opened (cache, TRUE, error)) - return NULL; - locker = g_mutex_locker_new (&priv->mutex); - - if (priv->floating) { - /* floating cache, check only the internal map */ - - GHashTableIter iter; - gpointer value; - - result = g_ptr_array_new_with_free_func (g_object_unref); - if (id == NULL) - return result; - - g_hash_table_iter_init (&iter, priv->cpt_map); - while (g_hash_table_iter_next (&iter, NULL, &value)) { - AsComponent *cpt = AS_COMPONENT (value); - if (g_strcmp0 (as_component_get_id (cpt), id) == 0) - g_ptr_array_add (result, g_object_ref (cpt)); - } - return result; - } - - txn = as_cache_transaction_new (cache, MDB_RDONLY, error); - if (txn == NULL) - return NULL; - - dval = as_cache_txn_get_value (cache, txn, priv->db_cids, id, &tmp_error); - if (tmp_error != NULL) { - g_propagate_error (error, tmp_error); - lmdb_transaction_abort (txn); - return NULL; - } - - result = as_cache_components_by_hash_list (cache, txn, dval.mv_data, dval.mv_size, error); - if (result == NULL) { - lmdb_transaction_abort (txn); - return NULL; - } else { - lmdb_transaction_commit (txn, NULL); - return result; - } -} - -/** - * as_cache_get_component_by_data_id: - * @cache: An instance of #AsCache. - * @cdid: The component data ID. - * @error: A #GError or %NULL. - * - * Retrieve component with the given data ID. - * - * Returns: (transfer full): An #AsComponent - */ -AsComponent* -as_cache_get_component_by_data_id (AsCache *cache, const gchar *cdid, GError **error) -{ - AsCachePrivate *priv = GET_PRIVATE (cache); - MDB_txn *txn; - MDB_val dval; - GError *tmp_error = NULL; - g_autofree guint8 *cpt_hash = NULL; - AsComponent *cpt; - g_autoptr(GMutexLocker) locker = NULL; - - if (!as_cache_check_opened (cache, TRUE, error)) - return NULL; - locker = g_mutex_locker_new (&priv->mutex); - - if (priv->floating) { - /* floating cache, check only the internal map */ - cpt = g_hash_table_lookup (priv->cpt_map, cdid); - return cpt? g_object_ref (cpt) : NULL; - } - - /* in case we "removed" the component on a readonly cache */ - as_generate_cache_checksum (cdid, -1, &cpt_hash, NULL); - if (g_hash_table_contains (priv->ro_removed_set, cpt_hash)) - return NULL; - - txn = as_cache_transaction_new (cache, MDB_RDONLY, error); - if (txn == NULL) - return NULL; - - dval = as_txn_get_value_by_hash (cache, txn, priv->db_cpts, cpt_hash, &tmp_error); - if (tmp_error != NULL) { - g_propagate_error (error, tmp_error); - lmdb_transaction_abort (txn); - return NULL; - } - if (dval.mv_size <= 0) { - /* nothing found? */ - lmdb_transaction_abort (txn); - return NULL; - } - - cpt = as_cache_component_from_dval (cache, txn, dval, error); - if (cpt == NULL) { - g_propagate_error (error, tmp_error); - lmdb_transaction_abort (txn); - return NULL; - } - - lmdb_transaction_commit (txn, NULL); - return cpt; -} - -/** - * as_cache_get_components_by_kind: - * @cache: An instance of #AsCache. - * @kind: The #AsComponentKind to retrieve. - * @error: A #GError or %NULL. - * - * Retrieve components of a specific kind. - * - * Returns: (transfer full): An array of #AsComponent - */ -GPtrArray* -as_cache_get_components_by_kind (AsCache *cache, AsComponentKind kind, GError **error) -{ - AsCachePrivate *priv = GET_PRIVATE (cache); - MDB_txn *txn; - GError *tmp_error = NULL; - MDB_val dval; - GPtrArray *result = NULL; - const gchar *kind_str = as_component_kind_to_string (kind); - g_autoptr(GMutexLocker) locker = NULL; - - if (!as_cache_check_opened (cache, FALSE, error)) - return NULL; - locker = g_mutex_locker_new (&priv->mutex); - - txn = as_cache_transaction_new (cache, MDB_RDONLY, error); - if (txn == NULL) - return NULL; - - dval = as_cache_txn_get_value (cache, txn, priv->db_kinds, kind_str, &tmp_error); - if (tmp_error != NULL) { - g_propagate_error (error, tmp_error); - lmdb_transaction_abort (txn); - return NULL; - } - - result = as_cache_components_by_hash_list (cache, txn, dval.mv_data, dval.mv_size, error); - if (result == NULL) { - lmdb_transaction_abort (txn); - return NULL; - } else { - lmdb_transaction_commit (txn, NULL); - return result; - } -} - -/** - * as_cache_get_components_by_provided_item: - * @cache: An instance of #AsCache. - * @kind: Kind of the provided item. - * @item: Name of the item. - * @error: A #GError or %NULL. - * - * Retrieve a list of components that provide a certain item. - * - * Returns: (transfer full): An array of #AsComponent - */ -GPtrArray* -as_cache_get_components_by_provided_item (AsCache *cache, AsProvidedKind kind, const gchar *item, GError **error) -{ - AsCachePrivate *priv = GET_PRIVATE (cache); - MDB_txn *txn; - GError *tmp_error = NULL; - MDB_val dval; - g_autofree gchar *item_key = NULL; - GPtrArray *result = NULL; - g_autoptr(GMutexLocker) locker = NULL; - - if (!as_cache_check_opened (cache, FALSE, error)) - return NULL; - locker = g_mutex_locker_new (&priv->mutex); - - item_key = g_strconcat (as_provided_kind_to_string (kind), item, NULL); - txn = as_cache_transaction_new (cache, MDB_RDONLY, error); - if (txn == NULL) - return NULL; - - dval = as_cache_txn_get_value (cache, txn, priv->db_provides, item_key, &tmp_error); - if (tmp_error != NULL) { - g_propagate_error (error, tmp_error); - lmdb_transaction_abort (txn); - return NULL; - } - - result = as_cache_components_by_hash_list (cache, txn, dval.mv_data, dval.mv_size, error); - if (result == NULL) { - lmdb_transaction_abort (txn); - return NULL; - } else { - lmdb_transaction_commit (txn, NULL); - return result; - } -} - -/** - * as_cache_get_components_by_categories: - * @cache: An instance of #AsCache. - * @categories: List of category names. - * @error: A #GError or %NULL. - * - * get a list of components in the selected categories. - * - * Returns: (transfer full): An array of #AsComponent - */ -GPtrArray* -as_cache_get_components_by_categories (AsCache *cache, gchar **categories, GError **error) -{ - AsCachePrivate *priv = GET_PRIVATE (cache); - MDB_txn *txn; - GError *tmp_error = NULL; - g_autoptr(GPtrArray) result = NULL; - g_autoptr(GMutexLocker) locker = NULL; - - if (!as_cache_check_opened (cache, FALSE, error)) - return NULL; - locker = g_mutex_locker_new (&priv->mutex); - - txn = as_cache_transaction_new (cache, MDB_RDONLY, error); - if (txn == NULL) - return NULL; - - result = g_ptr_array_new_with_free_func (g_object_unref); - for (guint i = 0; categories[i] != NULL; i++) { - g_autoptr(GPtrArray) tmp_res = NULL; - MDB_val dval; - - dval = as_cache_txn_get_value (cache, txn, priv->db_cats, categories[i], &tmp_error); - if (tmp_error != NULL) { - g_propagate_error (error, tmp_error); - lmdb_transaction_abort (txn); - return NULL; - } - if (dval.mv_size == 0) - continue; - - tmp_res = as_cache_components_by_hash_list (cache, txn, dval.mv_data, dval.mv_size, error); - if (tmp_res == NULL) { - lmdb_transaction_abort (txn); - return NULL; - } - - as_object_ptr_array_absorb (result, tmp_res); - } - - if (result == NULL) { - lmdb_transaction_abort (txn); - return NULL; - } else { - lmdb_transaction_commit (txn, NULL); - return g_steal_pointer (&result); - } -} - -/** - * as_cache_get_components_by_launchable: - * @cache: An instance of #AsCache. - * @kind: Type of the launchable. - * @id: ID of the launchable. - * @error: A #GError or %NULL. - * - * Get components which provide a certain launchable. - * - * Returns: (transfer full): An array of #AsComponent - */ -GPtrArray* -as_cache_get_components_by_launchable (AsCache *cache, AsLaunchableKind kind, const gchar *id, GError **error) -{ - AsCachePrivate *priv = GET_PRIVATE (cache); - MDB_txn *txn; - GError *tmp_error = NULL; - g_autofree gchar *entry_key = NULL; - MDB_val dval; - GPtrArray *result = NULL; - g_autoptr(GMutexLocker) locker = NULL; - - if (!as_cache_check_opened (cache, FALSE, error)) - return NULL; - locker = g_mutex_locker_new (&priv->mutex); - - entry_key = g_strconcat (as_launchable_kind_to_string (kind), id, NULL); - txn = as_cache_transaction_new (cache, MDB_RDONLY, error); - if (txn == NULL) - return NULL; - - dval = as_cache_txn_get_value (cache, txn, priv->db_launchables, entry_key, &tmp_error); - if (tmp_error != NULL) { - g_propagate_error (error, tmp_error); - lmdb_transaction_abort (txn); - return NULL; - } - - result = as_cache_components_by_hash_list (cache, txn, dval.mv_data, dval.mv_size, error); - if (result == NULL) { - lmdb_transaction_abort (txn); - return NULL; - } else { - lmdb_transaction_commit (txn, NULL); - return result; - } -} - -typedef struct { - AsComponent *cpt; - GPtrArray *matched_terms; - guint terms_found; -} AsSearchResultItem; - -static void -as_search_result_item_free (AsSearchResultItem *item) -{ - if (item->cpt != NULL) - g_object_unref (item->cpt); - if (item->matched_terms != NULL) - g_ptr_array_unref (item->matched_terms); - g_slice_free (AsSearchResultItem, item); -} - -/** - * as_cache_update_results_with_fts_value: - * - * Update results table using the full-text search scoring data from a GVariant dict - */ -static gboolean -as_cache_update_results_with_fts_value (AsCache *cache, MDB_txn *txn, - MDB_val dval, const gchar *matched_term, - GHashTable *results_ht, - gboolean exact_match, - GError **error) -{ - GError *tmp_error = NULL; - guint8 *data = NULL; - gsize data_len = 0; - const gsize ENTRY_LEN = sizeof(AsTokenType) + AS_CACHE_CHECKSUM_LEN * sizeof(guint8); - - data = dval.mv_data; - data_len = dval.mv_size; - - g_assert_cmpint (data_len % ENTRY_LEN, ==, 0); - if (data_len == 0) - return TRUE; - - for (gsize i = 0; i < data_len; i += ENTRY_LEN) { - guint sort_score; - AsSearchResultItem *sitem; - const guint8 *cpt_hash; - AsTokenType match_pval; - - cpt_hash = data + i; - match_pval = (AsTokenType) *(data + i + AS_CACHE_CHECKSUM_LEN); - - /* retrieve component with this hash */ - sitem = g_hash_table_lookup (results_ht, cpt_hash); - if (sitem == NULL) { - sitem = g_slice_new0 (AsSearchResultItem); - sitem->cpt = as_cache_component_by_hash (cache, txn, cpt_hash, &tmp_error); - sitem->matched_terms = g_ptr_array_new_with_free_func (g_free); - if (tmp_error != NULL) { - g_propagate_prefixed_error (error, tmp_error, "Failed to retrieve component data: "); - as_search_result_item_free (sitem); - return FALSE; - } - if (sitem->cpt == NULL) { - as_search_result_item_free (sitem); - } else { - sort_score = 0; - if (exact_match) - sort_score |= match_pval << 2; - else - sort_score |= match_pval; - - if ((as_component_get_kind (sitem->cpt) == AS_COMPONENT_KIND_ADDON) && (match_pval > 0)) - sort_score--; - - as_component_set_sort_score (sitem->cpt, sort_score); - sitem->terms_found = 1; - g_ptr_array_add (sitem->matched_terms, g_strdup (matched_term)); - -#if GLIB_CHECK_VERSION(2,68,0) - g_hash_table_insert (results_ht, - g_memdup2 (cpt_hash, AS_CACHE_CHECKSUM_LEN), - sitem); -#else - g_hash_table_insert (results_ht, - g_memdup (cpt_hash, AS_CACHE_CHECKSUM_LEN), - sitem); -#endif - - } - } else { - gboolean term_matched = FALSE; - - sort_score = as_component_get_sort_score (sitem->cpt); - if (exact_match) - sort_score |= match_pval << 2; - else - sort_score |= match_pval; - - if (as_component_get_sort_score (sitem->cpt) == 0) { - if ((as_component_get_kind (sitem->cpt) == AS_COMPONENT_KIND_ADDON)) - sort_score--; - } - - as_component_set_sort_score (sitem->cpt, sort_score); - - /* due to stemming and partial matches, we may match the same term a lot of times, - * but we only want to register a specific term match once (while still bumping up the - * search result's match score) */ - for (guint j = 0; j < sitem->matched_terms->len; j++) { - if (g_strcmp0 (matched_term, (const gchar *) g_ptr_array_index (sitem->matched_terms, j)) == 0) { - term_matched = TRUE; - break; - } - } - if (!term_matched) - sitem->terms_found++; - } - } - - return TRUE; -} - -/** - * as_cache_search_items_table_to_results: - * - * Converts the found items hash table entries to results list of components. - */ -static void -as_cache_search_items_table_to_results (GHashTable *results_ht, GPtrArray *results, guint required_terms_n) -{ - GHashTableIter ht_iter; - gpointer ht_value; - - if (g_hash_table_size (results_ht) == 0) - return; - - g_hash_table_iter_init (&ht_iter, results_ht); - while (g_hash_table_iter_next (&ht_iter, NULL, &ht_value)) { - AsSearchResultItem *sitem = (AsSearchResultItem*) ht_value; - if (sitem->terms_found < required_terms_n) - continue; - g_ptr_array_add (results, g_steal_pointer (&sitem->cpt)); - } - - /* the items hash table contents are invalid now */ - g_hash_table_remove_all (results_ht); -} - -/** - * as_cache_search: - * @cache: An instance of #AsCache. - * @terms: List of search terms - * @sort: %TRUE if results should be sorted by score. - * @error: A #GError or %NULL. - * - * Perform a search for the given terms. - * - * Returns: (transfer container): An array of #AsComponent - */ -GPtrArray* -as_cache_search (AsCache *cache, gchar **terms, gboolean sort, GError **error) -{ - AsCachePrivate *priv = GET_PRIVATE (cache); - MDB_txn *txn; - g_autoptr(GPtrArray) results = NULL; - g_autoptr(GHashTable) results_ht = NULL; - g_autoptr(GMutexLocker) locker = NULL; - - MDB_cursor *cur; - gint rc; - MDB_val dkey; - MDB_val dval; - g_autofree gsize *terms_lens = NULL; - guint terms_n = 0; - - if (!as_cache_check_opened (cache, FALSE, error)) - return NULL; - - if (terms == NULL) { - /* if we have no search terms, just return all components we have */ - return as_cache_get_components_all (cache, error); - } - - locker = g_mutex_locker_new (&priv->mutex); - - txn = as_cache_transaction_new (cache, MDB_RDONLY, error); - if (txn == NULL) - return NULL; - - results = g_ptr_array_new_with_free_func (g_object_unref); - results_ht = g_hash_table_new_full (as_cache_checksum_hash, - as_cache_checksum_equal, - g_free, - (GDestroyNotify) as_search_result_item_free); - - terms_n = g_strv_length (terms); - - /* unconditionally perform partial matching, which is slower, but yields more complete results - * closer to what the users seem to expect compared to the more narrow results we get when running - * an exact match query on the database */ - rc = mdb_cursor_open (txn, priv->db_fts, &cur); - if (rc != MDB_SUCCESS) { - g_set_error (error, - AS_CACHE_ERROR, - AS_CACHE_ERROR_FAILED, - "Unable to iterate cache (no cursor): %s", mdb_strerror (rc)); - lmdb_transaction_abort (txn); - return NULL; - } - - /* cache term string lengths */ - terms_lens = g_new0 (gsize, terms_n + 1); - for (guint i = 0; terms[i] != NULL; i++) - terms_lens[i] = strlen (terms[i]); - - rc = mdb_cursor_get (cur, &dkey, &dval, MDB_FIRST); - while (rc == 0) { - const gchar *matched_term = NULL; - const gchar *token = dkey.mv_data; - gsize token_len = dkey.mv_size; - - for (guint i = 0; terms[i] != NULL; i++) { - gsize term_len = terms_lens[i]; - /* if term length is bigger than the key, it will never match */ - if (term_len > dkey.mv_size) - continue; - - for (guint j = 0; j < token_len - term_len + 1; j++) { - if (strncmp (token + j, terms[i], term_len) == 0) { - /* partial match was successful */ - matched_term = terms[i]; - break; - } - } - if (matched_term != NULL) - break; - } - if (matched_term == NULL) { - rc = mdb_cursor_get(cur, &dkey, &dval, MDB_NEXT); - continue; - } + /* register the new section */ + g_ptr_array_add (priv->sections, + g_steal_pointer (&csec)); - /* we got a partial match, so add the components to our search result */ - if (dval.mv_size <= 0) - continue; - if (!as_cache_update_results_with_fts_value (cache, txn, - dval, matched_term, - results_ht, - FALSE, - error)) { - mdb_cursor_close (cur); - lmdb_transaction_abort (txn); - return NULL; - } + /* fix up section ordering */ + g_ptr_array_sort (priv->sections, as_cache_section_cmp); - rc = mdb_cursor_get(cur, &dkey, &dval, MDB_NEXT); - } - mdb_cursor_close (cur); + return ret; +} - /* compile our result */ - as_cache_search_items_table_to_results (results_ht, - results, - terms_n); +/** + * as_cache_set_contents_for_section: + * @cache: an #AsCache instance. + * + * Set cache contents for components belonging to the operating system, + * or other well-defined modules that AppStream can add automatically + * and that it knows a sensible sectoon key for. + * All previous contents of the cache section will be removed. + */ +gboolean +as_cache_set_contents_for_section (AsCache *cache, + AsComponentScope scope, + AsFormatStyle source_format_style, + gboolean is_os_data, + GPtrArray *cpts, + const gchar *cache_key, + gpointer refinefn_user_data, + GError **error) +{ + return as_cache_set_contents_internal (cache, + scope, + source_format_style, + is_os_data, + cpts, + cache_key, + refinefn_user_data, + error); +} - /* we don't need the mutex anymore, no class member access here */ - g_clear_pointer (&locker, g_mutex_locker_free); +/** + * as_cache_set_contents_for_path: + * @cache: an #AsCache instance. + * + * Set cache contents for components added by the API user, which may be in + * completely arbitrary paths. + * All previous contents of the cache section will be removed. + */ +gboolean +as_cache_set_contents_for_path (AsCache *cache, + GPtrArray *cpts, + const gchar *filename, + gpointer refinefn_user_data, + GError **error) +{ + if (g_strcmp0 (filename, "os-catalog") == 0 || g_strcmp0 (filename, "flatpak") == 0 || g_strcmp0 (filename, "metainfo") == 0) { + g_set_error (error, + AS_CACHE_ERROR, + AS_CACHE_ERROR_BAD_VALUE, + "Can not add extra repository data with reserved cache key name '%s'.", filename); + return FALSE; - /* sort the results by their priority */ - if (sort) - as_sort_components_by_score (results); + } - lmdb_transaction_commit (txn, NULL); - return g_steal_pointer (&results); + return as_cache_set_contents_internal (cache, + as_utils_guess_scope_from_path (filename), + AS_FORMAT_STYLE_COLLECTION, + FALSE, /* no OS data */ + cpts, + filename, + refinefn_user_data, + error); } /** - * as_cache_has_component_id: + * as_cache_get_ctime_with_section_key: * @cache: An instance of #AsCache. - * @id: The component ID to search for. - * @error: A #GError or %NULL. + * @scope: Component scope + * @section_key: An internal cache section key. + * @cache_scope: Returns which scope the cache that was most recent is in. * - * Check if there is any component(s) with the given ID in the cache. - * - * Returns: %TRUE if the ID is known. + * Returns: ctime of the cache section. */ -gboolean -as_cache_has_component_id (AsCache *cache, const gchar *id, GError **error) +static time_t +as_cache_get_ctime_with_section_key (AsCache *cache, AsComponentScope scope, const gchar *section_key, AsCacheScope *cache_scope) { AsCachePrivate *priv = GET_PRIVATE (cache); - MDB_txn *txn; - GError *tmp_error = NULL; - MDB_val dval; - gboolean found; - g_autoptr(GMutexLocker) locker = NULL; + struct stat cache_sbuf; + g_autofree gchar *xb_fname_user = NULL; + g_autofree gchar *xb_fname_sys = NULL; + time_t t_sys, t_user; + + xb_fname_user = as_cache_get_section_fname_for (cache, + AS_CACHE_SCOPE_WRITABLE, + scope, + section_key); + if (g_strcmp0 (priv->cache_root_dir, priv->system_cache_dir) != 0) { + xb_fname_sys = as_cache_get_section_fname_for (cache, + AS_CACHE_SCOPE_SYSTEM, + scope, + section_key); + } + + if (cache_scope != NULL) + *cache_scope = AS_CACHE_SCOPE_WRITABLE; + + t_user = 0; + t_sys = 0; + if (stat (xb_fname_user, &cache_sbuf) == 0) + t_user = cache_sbuf.st_ctime; + if (xb_fname_sys == NULL) { + return t_user; + } else { + if (stat (xb_fname_sys, &cache_sbuf) == 0) + t_sys = cache_sbuf.st_ctime; + if (t_sys > t_user) { + if (cache_scope != NULL) + *cache_scope = AS_CACHE_SCOPE_SYSTEM; + return t_sys; + } + return t_user; + } +} - if (!as_cache_check_opened (cache, TRUE, error)) - return FALSE; - locker = g_mutex_locker_new (&priv->mutex); +/** + * as_cache_get_ctime: + * @cache: An instance of #AsCache. + * @scope: Component scope + * @cache_key: A user-provided cache key. + * @cache_scope: returns which scope the cache that was most recent is in. + * + * Returns: ctime of the cache section. + */ +time_t +as_cache_get_ctime (AsCache *cache, AsComponentScope scope, const gchar *cache_key, AsCacheScope *cache_scope) +{ + g_autofree gchar *section_key = NULL; - if (priv->floating) { - /* floating cache, check only the internal map */ - return g_hash_table_contains (priv->cid_set, id); + if (scope == AS_COMPONENT_SCOPE_UNKNOWN) { + scope = AS_COMPONENT_SCOPE_SYSTEM; + if (g_str_has_prefix (cache_key, "/home") || g_str_has_prefix (cache_key, g_get_home_dir ())) + scope = AS_COMPONENT_SCOPE_USER; } - txn = as_cache_transaction_new (cache, MDB_RDONLY, error); - if (txn == NULL) - return FALSE; + section_key = as_cache_build_section_key (cache, cache_key); + return as_cache_get_ctime_with_section_key (cache, scope, section_key, cache_scope); +} - dval = as_cache_txn_get_value (cache, txn, priv->db_cids, id, &tmp_error); - if (tmp_error != NULL) { - g_propagate_error (error, tmp_error); - lmdb_transaction_abort (txn); - return FALSE; - } +/** + * as_cache_check_section_outdated: + * + * Returns: %TRUE if ctime of @origin_path is newer than the ctime on the cache section. + */ +static gboolean +as_cache_check_section_outdated (AsCache *cache, AsComponentScope scope, const gchar *cache_key, const gchar *origin_path) +{ + struct stat sb; + time_t cache_ctime; + + if (stat (origin_path, &sb) < 0) + return TRUE; - found = dval.mv_size > 0; + cache_ctime = as_cache_get_ctime (cache, scope, cache_key, NULL); + if (sb.st_ctime > cache_ctime) + return TRUE; - lmdb_transaction_commit (txn, NULL); - return found; + return FALSE; } /** - * as_cache_count_components: - * @cache: An instance of #AsCache. - * @error: A #GError or %NULL. - * - * Get the amount of components the cache holds. + * as_cache_load_section_internal: * - * Returns: Components count in the database, or -1 on error. + * Load cache section. */ -gssize -as_cache_count_components (AsCache *cache, GError **error) +static void +as_cache_load_section_internal (AsCache *cache, + AsComponentScope scope, + const gchar *cache_key, + AsFormatStyle source_format_style, + gboolean is_os_data, + gboolean *is_outdated, + gpointer refine_user_data) { AsCachePrivate *priv = GET_PRIVATE (cache); - MDB_txn *txn; - MDB_stat stats; - gint rc; - gssize count = -1; + AsCacheScope cache_scope; + g_autofree gchar *section_key = NULL; + g_autofree gchar *internal_section_key = NULL; + g_autofree gchar *xb_fname = NULL; g_autoptr(GMutexLocker) locker = NULL; + g_autoptr(AsCacheSection) csec = NULL; + g_autoptr(GFile) file = NULL; + g_autoptr(GError) tmp_error = NULL; + + section_key = as_cache_build_section_key (cache, cache_key); + internal_section_key = g_strconcat (as_component_scope_to_string (scope), ":", section_key, NULL); + + /* check which available cache is the most recent one */ + as_cache_get_ctime_with_section_key (cache, + scope, + section_key, + &cache_scope); + + xb_fname = as_cache_get_section_fname_for (cache, + cache_scope, + scope, + section_key); + + if (!g_file_test (xb_fname, G_FILE_TEST_EXISTS)) { + /* nothing to do if no cache file exists that we can load */ + if (is_outdated != NULL) + *is_outdated = TRUE; + return; + } - if (!as_cache_check_opened (cache, FALSE, error)) - return 0; - locker = g_mutex_locker_new (&priv->mutex); + locker = g_mutex_locker_new (&priv->sec_mutex); + + csec = as_cache_section_new (internal_section_key); + csec->is_os_data = is_os_data && scope == AS_COMPONENT_SCOPE_SYSTEM; + csec->scope = scope; + csec->format_style = source_format_style; + csec->fname = g_strdup (xb_fname); + csec->refine_func_udata = refine_user_data; + + csec->silo = xb_silo_new (); + file = g_file_new_for_path (csec->fname); + if (!xb_silo_load_from_file (csec->silo, + file, + XB_SILO_LOAD_FLAG_NONE, + NULL, + &tmp_error)) { + g_warning ("Failed to load AppStream cache section '%s' - marking cache as outdated. Error: %s", + internal_section_key, tmp_error->message); + if (is_outdated != NULL) + *is_outdated = TRUE; + return; - txn = as_cache_transaction_new (cache, MDB_RDONLY, error); - if (txn == NULL) - return 0; + } - rc = mdb_stat(txn, priv->db_cpts, &stats); - if (rc == MDB_SUCCESS) { - count = stats.ms_entries; - } else { - g_set_error (error, - AS_CACHE_ERROR, - AS_CACHE_ERROR_FAILED, - "Unable to retrieve cache statistics: %s", mdb_strerror (rc)); + /* register the new section, replacing any old data */ + for (guint i = 0; i < priv->sections->len; i++) { + AsCacheSection *csec_entry = g_ptr_array_index (priv->sections, i); + if (g_strcmp0 (csec_entry->key, internal_section_key) == 0) { + g_ptr_array_remove_index_fast (priv->sections, i); + break; + } } + g_ptr_array_add (priv->sections, + g_steal_pointer (&csec)); + g_debug ("Using cache file: %s", xb_fname); - lmdb_transaction_commit (txn, NULL); - return count; + /* fix up section ordering */ + g_ptr_array_sort (priv->sections, as_cache_section_cmp); } - - /** - * as_cache_get_ctime: - * @cache: An instance of #AsCache. + * as_cache_load_section_for_key: + * @cache: an #AsCache instance. * - * Returns: ctime of the cache database. + * Load cache section that was created using a custom, special key. */ -time_t -as_cache_get_ctime (AsCache *cache) +void +as_cache_load_section_for_key (AsCache *cache, + AsComponentScope scope, + AsFormatStyle source_format_style, + gboolean is_os_data, + const gchar *cache_key, + gboolean *is_outdated, + gpointer refinefn_user_data) { - AsCachePrivate *priv = GET_PRIVATE (cache); - struct stat cache_sbuf; - g_autoptr(GMutexLocker) locker = g_mutex_locker_new (&priv->mutex); - - if (priv->fname == NULL) - return 0; - - if (stat (priv->fname, &cache_sbuf) < 0) - return 0; - else - return cache_sbuf.st_ctime; + as_cache_load_section_internal (cache, + scope, + cache_key, + source_format_style, + is_os_data, + is_outdated, + refinefn_user_data); } /** - * as_cache_is_open: - * @cache: An instance of #AsCache. + * as_cache_load_section_for_path: + * @cache: an #AsCache instance. * - * Returns: %TRUE if the cache is open. + * Try to load possibly cached data for a generic, custom path. */ -gboolean -as_cache_is_open (AsCache *cache) +void +as_cache_load_section_for_path (AsCache *cache, + const gchar *filename, + gboolean *is_outdated, + gpointer refinefn_user_data) { - AsCachePrivate *priv = GET_PRIVATE (cache); - g_autoptr(GMutexLocker) locker = g_mutex_locker_new (&priv->mutex); - return priv->opened; + AsComponentScope scope; + + scope = as_utils_guess_scope_from_path (filename); + if (is_outdated != NULL) + *is_outdated = as_cache_check_section_outdated (cache, scope, filename, filename); + + as_cache_load_section_internal (cache, + scope, + filename, + AS_FORMAT_STYLE_COLLECTION, + FALSE, + is_outdated, + refinefn_user_data); } /** - * as_cache_make_floating: - * @cache: An instance of #AsCache. + * as_cache_mask_by_data_id: + * @cache: an #AsCache instance. + * @cdid: Data-ID of an #AsComponent * - * Make cache "floating": Only some actions are permitted and nothing - * is written to disk until the floating state is changed. + * Mark a component as "deleted" from the cache. + * The component will no longer show up in search results or be returned + * by other queries. */ void -as_cache_make_floating (AsCache *cache) +as_cache_mask_by_data_id (AsCache *cache, const gchar *cdid) { AsCachePrivate *priv = GET_PRIVATE (cache); - g_autoptr(GMutexLocker) locker = g_mutex_locker_new (&priv->mutex); - - if (priv->floating) - return; - priv->floating = TRUE; - - g_hash_table_remove_all (priv->cpt_map); - g_hash_table_remove_all (priv->cid_set); - g_debug ("Cache set to floating mode."); + g_autoptr(GMutexLocker) locker = g_mutex_locker_new (&priv->sec_mutex); + g_hash_table_insert (priv->masked, + g_strdup (cdid), + GINT_TO_POINTER (TRUE)); } /** - * as_cache_unfloat: - * @cache: An instance of #AsCache. + * as_cache_add_masking_component: + * @cache: an #AsCache instance. + * @cpts: An array of #AsComponent to add as override * - * Persist all changes from a floating cache. - * Return the number of invalid components. + * Add a masking component to the cache that is returned in place of + * existing components in the cache. */ -guint -as_cache_unfloat (AsCache *cache, GError **error) +gboolean +as_cache_add_masking_components (AsCache *cache, GPtrArray *cpts, GError **error) { AsCachePrivate *priv = GET_PRIVATE (cache); - GHashTableIter iter; - gpointer ht_value; - guint invalid_cpts = 0; - - g_mutex_lock (&priv->mutex); - - priv->floating = FALSE; - - g_hash_table_iter_init (&iter, priv->cpt_map); - while (g_hash_table_iter_next (&iter, NULL, &ht_value)) { - AsComponent *cpt = AS_COMPONENT (ht_value); - - /* validate the component */ - if (!as_component_is_valid (cpt)) { - /* we still succeed if the components originates from a .desktop file - - * we care less about them and they generally have bad quality, so some issues - * pop up on pretty much every system */ - if (as_component_get_origin_kind (cpt) == AS_ORIGIN_KIND_DESKTOP_ENTRY) { - g_debug ("Ignored '%s': The component (from a .desktop file) is invalid.", as_component_get_id (cpt)); - } else { - g_debug ("WARNING: Ignored component '%s': The component is invalid.", as_component_get_id (cpt)); - invalid_cpts++; + g_autoptr(AsCacheSection) old_mcsec = NULL; + g_autoptr(AsCacheSection) mcsec = NULL; + g_autoptr(GPtrArray) cpts_final = NULL; + g_autoptr(GFile) file = NULL; + g_autoptr(GError) tmp_error = NULL; + g_autofree gchar *volatile_fname = NULL; + gint fd; + g_autoptr(GMutexLocker) locker = g_mutex_locker_new (&priv->sec_mutex); + + /* find old masking section */ + for (guint i = 0; i < priv->sections->len; i++) { + AsCacheSection *csec_entry = g_ptr_array_index (priv->sections, i); + if (csec_entry->is_mask) { + old_mcsec = g_ptr_array_steal_index_fast (priv->sections, i); + break; + } + } + + cpts_final = g_ptr_array_new_with_free_func (g_object_unref); + if (old_mcsec != NULL) { + g_autoptr(GPtrArray) array = NULL; + + /* retrieve the old data */ + array = xb_silo_query (old_mcsec->silo, + "components/component", 0, + NULL); + if (array != NULL) { + for (guint j = 0; j < array->len; j++) { + g_autoptr (AsComponent) cpt = NULL; + gpointer iptr; + XbNode *node = g_ptr_array_index (array, j); + + cpt = as_cache_component_from_node (cache, + old_mcsec, + node, + NULL); + if (cpt == NULL) + continue; + + /* just delete masked components */ + iptr = g_hash_table_lookup (priv->masked, as_component_get_data_id (cpt)); + if (iptr != NULL && GPOINTER_TO_INT (iptr) == TRUE) + continue; + + g_ptr_array_add (cpts_final, + g_steal_pointer (&cpt)); + g_hash_table_insert (priv->masked, + g_strdup (as_component_get_data_id (cpt)), + GINT_TO_POINTER (FALSE)); } - continue; } - g_mutex_unlock (&priv->mutex); - if (!as_cache_insert (cache, cpt, error)) - return 0; - g_mutex_lock (&priv->mutex); + /* drop the old data */ + as_cache_remove_section_file (cache, old_mcsec); + } + + /* generate filename for the volatile section in memory */ + volatile_fname = g_build_filename (g_get_user_runtime_dir (), "appstream-extra-XXXXXX.mdb", NULL); + fd = g_mkstemp (volatile_fname); + if (fd > 0) + close (fd); + + /* create new section */ + mcsec = as_cache_section_new ("memory:volatile_mask"); + mcsec->scope = AS_COMPONENT_SCOPE_USER; + mcsec->format_style = AS_FORMAT_STYLE_COLLECTION; + mcsec->fname = g_steal_pointer (&volatile_fname); + mcsec->is_mask = TRUE; + + /* create final component set */ + for (guint i = 0; i < cpts->len; i++) { + AsComponent *cpt = g_ptr_array_index (cpts, i); + g_ptr_array_add (cpts_final, + g_object_ref (cpt)); + g_hash_table_insert (priv->masked, + g_strdup (as_component_get_data_id (cpt)), + GINT_TO_POINTER (FALSE)); + } + + mcsec->silo = as_cache_components_to_internal_xb (cache, + cpts_final, + FALSE, /* do not refine */ + NULL, + &tmp_error); + if (mcsec->silo == NULL) { + g_propagate_prefixed_error (error, + g_steal_pointer (&tmp_error), + "Unable to add masking components to cache: Silo build failed. "); + return FALSE; + } + + /* write data */ + file = g_file_new_for_path (mcsec->fname); + if (!xb_silo_save_to_file (mcsec->silo, file, NULL, &tmp_error)) { + g_propagate_prefixed_error (error, + g_steal_pointer (&tmp_error), + "Unable to add masking components to cache: Failed to store silo. "); + return FALSE; } - g_hash_table_remove_all (priv->cid_set); - g_hash_table_remove_all (priv->cpt_map); + /* register the new section */ + g_ptr_array_add (priv->sections, + g_steal_pointer (&mcsec)); - g_mutex_unlock (&priv->mutex); - g_debug ("Cache returned from floating mode (all changes are now persistent)"); + /* fix up section ordering */ + g_ptr_array_sort (priv->sections, as_cache_section_cmp); - return invalid_cpts; + return TRUE; } /** - * as_cache_get_location: + * as_cache_set_refine_func: * @cache: An instance of #AsCache. * - * Returns: Location string for this cache. + * Set function to be called on a component after it was deserialized. */ -const gchar* -as_cache_get_location (AsCache *cache) +void +as_cache_set_refine_func (AsCache *cache, AsCacheDataRefineFn func) { AsCachePrivate *priv = GET_PRIVATE (cache); - g_autoptr(GMutexLocker) locker = g_mutex_locker_new (&priv->mutex); - return priv->fname; + priv->cpt_refine_func = func; } /** - * as_cache_set_location: - * @cache: An instance of #AsCache. - * - * Set location string for this database. + * as_cache_query_components_internal: */ -void -as_cache_set_location (AsCache *cache, const gchar *location) +static GPtrArray* +as_cache_query_components (AsCache *cache, + const gchar *xpath, + XbQueryContext *context, + guint limit, + GError **error) { AsCachePrivate *priv = GET_PRIVATE (cache); - g_autoptr(GMutexLocker) locker = g_mutex_locker_new (&priv->mutex); + g_autoptr(GPtrArray) results = NULL; + g_autoptr(GHashTable) results_map = NULL; + g_autoptr(GHashTable) known_os_cids = NULL; + GHashTableIter ht_iter; + gpointer ht_value; + + results_map = g_hash_table_new_full (g_str_hash, g_str_equal, + g_free, g_object_unref); + known_os_cids = g_hash_table_new_full (g_str_hash, g_str_equal, + g_free, NULL); + + for (guint i = 0; i < priv->sections->len; i++) { + g_autoptr(GPtrArray) array = NULL; + g_autoptr(GError) tmp_error = NULL; + g_autoptr(XbQuery) query = NULL; + AsCacheSection *csec = (AsCacheSection*) g_ptr_array_index (priv->sections, i); + + g_debug ("Querying `%s` in %s", xpath, csec->key); + query = xb_query_new (csec->silo, xpath, &tmp_error); + if (query == NULL) { + g_propagate_prefixed_error (error, + g_steal_pointer (&tmp_error), + "Unable to construct query: "); + return NULL; + } + + array = xb_silo_query_with_context (csec->silo, + query, + context, + &tmp_error); + if (array == NULL) { + if (g_error_matches (tmp_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) + continue; + if (g_error_matches (tmp_error, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT)) + continue; + g_propagate_prefixed_error (error, + g_steal_pointer (&tmp_error), + "Unable to run query: "); + return NULL; + } + + for (guint j = 0; j < array->len; j++) { + g_autoptr (AsComponent) cpt = NULL; + gchar *tmp; + XbNode *node = g_ptr_array_index (array, j); + + if (csec->is_os_data && csec->format_style == AS_FORMAT_STYLE_METAINFO) { + const gchar *cid = xb_node_query_text (node, "id", NULL); + if (g_hash_table_contains (known_os_cids, cid) && + !priv->prefer_os_metainfo) + continue; + } - g_free (priv->fname); - priv->fname = g_strdup (location); + cpt = as_cache_component_from_node (cache, + csec, + node, + error); + if (cpt == NULL) + return NULL; + + /* don't display masked components */ + if (!csec->is_mask && g_hash_table_contains (priv->masked, as_component_get_data_id (cpt))) + continue; + + if (csec->is_os_data) + g_hash_table_add (known_os_cids, + g_strdup (as_component_get_id (cpt))); + + tmp = g_strdup (as_component_get_data_id (cpt)); + g_hash_table_insert (results_map, + tmp, + g_steal_pointer (&cpt)); + } + } + + results = g_ptr_array_new_with_free_func (g_object_unref); + g_hash_table_iter_init (&ht_iter, results_map); + while (g_hash_table_iter_next (&ht_iter, NULL, &ht_value)) + g_ptr_array_add (results, g_object_ref (ht_value)); + + return g_steal_pointer (&results); } /** - * as_cache_get_nosync: + * as_cache_get_components_all: * @cache: An instance of #AsCache. + * @error: A #GError or %NULL. + * + * Retrieve all components this cache contains. * - * Returns: %TRUE if we don't sync explicitly. + * Returns: (transfer full): An array of #AsComponent */ -gboolean -as_cache_get_nosync (AsCache *cache) +GPtrArray* +as_cache_get_components_all (AsCache *cache, GError **error) { - AsCachePrivate *priv = GET_PRIVATE (cache); - g_autoptr(GMutexLocker) locker = g_mutex_locker_new (&priv->mutex); - return priv->nosync; + return as_cache_query_components (cache, + "components/component", + NULL, + 0, + error); } /** - * as_cache_set_nosync: + * as_cache_get_components_by_id: * @cache: An instance of #AsCache. + * @id: The component ID to search for. + * @error: A #GError or %NULL. + * + * Retrieve components with the selected ID. * - * Set whether the cache should sync to disk explicitly or not. - * This setting may be ignored if the cache is in temporary - * or in-memory mode. + * Returns: (transfer full): An array of #AsComponent */ -void -as_cache_set_nosync (AsCache *cache, gboolean nosync) +GPtrArray* +as_cache_get_components_by_id (AsCache *cache, const gchar *id, GError **error) { - AsCachePrivate *priv = GET_PRIVATE (cache); - g_autoptr(GMutexLocker) locker = g_mutex_locker_new (&priv->mutex); - priv->nosync = nosync; + g_auto(XbQueryContext) context = XB_QUERY_CONTEXT_INIT (); + xb_value_bindings_bind_str (xb_query_context_get_bindings (&context), 0, id, NULL); + return as_cache_query_components (cache, + "components/component/id[text()=?]/..", + &context, + 0, + error); } /** - * as_cache_get_readonly: + * as_cache_get_components_by_extends: * @cache: An instance of #AsCache. + * @extends_id: The component ID to find extensions for. + * @error: A #GError or %NULL. + * + * Retrieve components extending a component with the selected ID. * - * Returns: %TRUE if the cache is read-only. + * Returns: (transfer full): An array of #AsComponent */ -gboolean -as_cache_get_readonly (AsCache *cache) +GPtrArray* +as_cache_get_components_by_extends (AsCache *cache, const gchar *extends_id, GError **error) { - AsCachePrivate *priv = GET_PRIVATE (cache); - g_autoptr(GMutexLocker) locker = g_mutex_locker_new (&priv->mutex); - return priv->readonly; + g_auto(XbQueryContext) context = XB_QUERY_CONTEXT_INIT (); + xb_value_bindings_bind_str (xb_query_context_get_bindings (&context), 0, extends_id, NULL); + return as_cache_query_components (cache, + "components/component/extends[text()=?]/..", + &context, + 0, + error); } /** - * as_cache_set_readonly: + * as_cache_get_components_by_kind: * @cache: An instance of #AsCache. + * @kind: The #AsComponentKind to retrieve. + * @error: A #GError or %NULL. + * + * Retrieve components of a specific kind. * - * Set whether the cache should be read-only. + * Returns: (transfer full): An array of #AsComponent */ -void -as_cache_set_readonly (AsCache *cache, gboolean ro) +GPtrArray* +as_cache_get_components_by_kind (AsCache *cache, AsComponentKind kind, GError **error) { - AsCachePrivate *priv = GET_PRIVATE (cache); - g_autoptr(GMutexLocker) locker = g_mutex_locker_new (&priv->mutex); - priv->readonly = ro; + g_auto(XbQueryContext) context = XB_QUERY_CONTEXT_INIT (); + xb_value_bindings_bind_str (xb_query_context_get_bindings (&context), + 0, + as_component_kind_to_string (kind), + NULL); + return as_cache_query_components (cache, + "components/component[@type=?]", + &context, + 0, + error); } /** - * as_cache_set_refine_func: + * as_cache_get_components_by_provided_item: * @cache: An instance of #AsCache. + * @kind: Kind of the provided item. + * @item: Name of the item. + * @error: A #GError or %NULL. * - * Set function to be called on a component after it was deserialized. + * Retrieve a list of components that provide a certain item. + * + * Returns: (transfer full): An array of #AsComponent */ -void -as_cache_set_refine_func (AsCache *cache, GFunc func, gpointer user_data) +GPtrArray* +as_cache_get_components_by_provided_item (AsCache *cache, AsProvidedKind kind, const gchar *item, GError **error) { - AsCachePrivate *priv = GET_PRIVATE (cache); - g_autoptr(GMutexLocker) locker = g_mutex_locker_new (&priv->mutex); + const gchar *kind_node_name = NULL; + const gchar *xpath_query_tmpl = NULL; + const gchar *xpath_query_type_tmpl = NULL; + g_autofree gchar *xpath_query = NULL; + g_auto(XbQueryContext) context = XB_QUERY_CONTEXT_INIT (); + XbValueBindings *vbindings = xb_query_context_get_bindings (&context); + + xpath_query_tmpl = "components/component/provides/%s[text()=?]/../.."; + xpath_query_type_tmpl = "components/component/provides/%s[text()=?][@type='%s']/../.."; + + if (kind == AS_PROVIDED_KIND_LIBRARY) + kind_node_name = "library"; + else if (kind == AS_PROVIDED_KIND_BINARY) + kind_node_name = "binary"; + else if (kind == AS_PROVIDED_KIND_DBUS_SYSTEM) + xpath_query = g_strdup_printf (xpath_query_type_tmpl, "dbus", "system"); + else if (kind == AS_PROVIDED_KIND_DBUS_USER) + xpath_query = g_strdup_printf (xpath_query_type_tmpl, "dbus", "user"); + else if (kind == AS_PROVIDED_KIND_FIRMWARE_RUNTIME) + xpath_query = g_strdup_printf (xpath_query_type_tmpl, "firmware", "runtime"); + else if (kind == AS_PROVIDED_KIND_FIRMWARE_FLASHED) + xpath_query = g_strdup_printf (xpath_query_type_tmpl, "firmware", "flashed"); + else + kind_node_name = as_provided_kind_to_string (kind); - priv->cpt_refine_func = func; - priv->cpt_refine_func_udata = user_data; + if (xpath_query == NULL) + xpath_query = g_strdup_printf (xpath_query_tmpl, kind_node_name); + xb_value_bindings_bind_str (vbindings, 0, + item, + NULL); + return as_cache_query_components (cache, + xpath_query, + &context, + 0, + error); } /** - * as_cache_error_quark: + * as_cache_get_components_by_categories: + * @cache: An instance of #AsCache. + * @categories: List of category names. + * @error: A #GError or %NULL. * - * Return value: An error quark. - **/ -GQuark -as_cache_error_quark (void) + * get a list of components in the selected categories. + * + * Returns: (transfer full): An array of #AsComponent + */ +GPtrArray* +as_cache_get_components_by_categories (AsCache *cache, gchar **categories, GError **error) { - static GQuark quark = 0; - if (!quark) - quark = g_quark_from_static_string ("AsCache"); - return quark; + g_autoptr(GString) xpath = NULL; + g_auto(XbQueryContext) context = XB_QUERY_CONTEXT_INIT (); + XbValueBindings *vbindings = xb_query_context_get_bindings (&context); + + if (categories == NULL || categories[0] == NULL) + return g_ptr_array_new_with_free_func (g_object_unref); + + xpath = g_string_new ("components/component/categories"); + for (guint i = 0; categories[i] != NULL; i++) { + g_string_append (xpath, "/category[text()=?]/.."); + xb_value_bindings_bind_str (vbindings, i, + categories[i], + NULL); + } + g_string_append (xpath, "/.."); + + return as_cache_query_components (cache, + xpath->str, + &context, + 0, + error); } /** - * as_cache_new: + * as_cache_get_components_by_launchable: + * @cache: An instance of #AsCache. + * @kind: Type of the launchable. + * @id: ID of the launchable. + * @error: A #GError or %NULL. * - * Creates a new #AsCache. + * Get components which provide a certain launchable. * - * Returns: (transfer full): a #AsCache + * Returns: (transfer full): An array of #AsComponent + */ +GPtrArray* +as_cache_get_components_by_launchable (AsCache *cache, AsLaunchableKind kind, const gchar *id, GError **error) +{ + g_auto(XbQueryContext) context = XB_QUERY_CONTEXT_INIT (); + g_autofree gchar *xpath = NULL; + xb_value_bindings_bind_str (xb_query_context_get_bindings (&context), + 0, id, NULL); + xpath = g_strdup_printf ("components/component/launchable[@type='%s'][text()=?]/..", as_launchable_kind_to_string (kind)); + return as_cache_query_components (cache, + xpath, + &context, + 0, + error); +} + +/** + * as_cache_search: + * @cache: An instance of #AsCache. + * @terms: List of search terms + * @sort: %TRUE if results should be sorted by score. + * @error: A #GError or %NULL. * - **/ -AsCache* -as_cache_new (void) + * Perform a search for the given terms. + * + * Returns: (transfer container): An array of #AsComponent + */ +GPtrArray* +as_cache_search (AsCache *cache, gchar **terms, gboolean sort, GError **error) { - AsCache *cache; - cache = g_object_new (AS_TYPE_CACHE, NULL); - return AS_CACHE (cache); + g_autoptr(GString) xpath = NULL; + GPtrArray *results; + g_auto(XbQueryContext) context = XB_QUERY_CONTEXT_INIT (); + XbValueBindings *vbindings = xb_query_context_get_bindings (&context); + + if (terms == NULL || terms[0] == NULL) + return g_ptr_array_new_with_free_func (g_object_unref); + + xpath = g_string_new ("components/component/__asi_tokens"); + for (guint i = 0; terms[i] != NULL; i++) { + g_string_append (xpath, "/t[text()~=?]/.."); + xb_value_bindings_bind_str (vbindings, i, + terms[i], + NULL); + } + g_string_append (xpath, "/.."); + + results = as_cache_query_components (cache, + xpath->str, + &context, + 0, + error); + + /* sort the results by their priority */ + if (sort) + as_sort_components_by_score (results); + + return results; } diff --git a/src/as-cache.h b/src/as-cache.h index a704dd262..0e6595f3f 100644 --- a/src/as-cache.h +++ b/src/as-cache.h @@ -45,13 +45,42 @@ struct _AsCacheClass void (*_as_reserved6) (void); }; +/** + * AsCacheScope: + * @AS_CACHE_SCOPE_UNKNOWN: Unknown scope, or scope is not relevant. + * @AS_CACHE_SCOPE_SYSTEM: System-wide cache, always considered unwritable. + * @AS_CACHE_SCOPE_WRITABLE: User-specific cache, always writable. + * + * Scope of the cache. + **/ +typedef enum { + AS_CACHE_SCOPE_UNKNOWN, + AS_CACHE_SCOPE_SYSTEM, + AS_CACHE_SCOPE_WRITABLE +} AsCacheScope; + +/** + * AsCacheDataRefineFn: + * @cpt: (not nullable): The component that should be refined. + * @is_serialization: %TRUE on serialization to the cache. + * @user_data: Additional data. + * + * Helper function called by #AsCache to refine components and add additional + * data to them, transform data to be suitable for storing or unpack it once + * the component is loaded from cache again. + */ +typedef void (*AsCacheDataRefineFn)(AsComponent *cpt, + gboolean is_serialization, + gpointer user_data); + /** * AsCacheError: * @AS_CACHE_ERROR_FAILED: Generic failure + * @AS_CACHE_ERROR_PERMISSIONS: Some permissions are missing, e.g. a file may not be writable + * @AS_CACHE_ERROR_BAD_VALUE: Some value, possibly user-defined, was invalid. * @AS_CACHE_ERROR_NOT_OPEN: Cache was not open. * @AS_CACHE_ERROR_WRONG_FORMAT: Cache has an unsupported format version. * @AS_CACHE_ERROR_LOCALE_MISMATCH: Cache locale was different from the expected one. - * @AS_CACHE_ERROR_FLOATING: The given action can not be performed on a floating cache. * @AS_CACHE_ERROR_NO_FILENAME: No filename was set to open the database. * @AS_CACHE_ERROR_BAD_DATA: The data that should be added failed a sanity check * @@ -59,10 +88,11 @@ struct _AsCacheClass **/ typedef enum { AS_CACHE_ERROR_FAILED, + AS_CACHE_ERROR_PERMISSIONS, + AS_CACHE_ERROR_BAD_VALUE, AS_CACHE_ERROR_NOT_OPEN, AS_CACHE_ERROR_WRONG_FORMAT, AS_CACHE_ERROR_LOCALE_MISMATCH, - AS_CACHE_ERROR_FLOATING, AS_CACHE_ERROR_NO_FILENAME, AS_CACHE_ERROR_BAD_DATA, /*< private >*/ @@ -72,82 +102,97 @@ typedef enum { #define AS_CACHE_ERROR as_cache_error_quark () GQuark as_cache_error_quark (void); -AsCache *as_cache_new (void); - -gboolean as_cache_open (AsCache *cache, - const gchar *fname, - const gchar *locale, - GError **error); -gboolean as_cache_open2 (AsCache *cache, - const gchar *locale, - GError **error); -gboolean as_cache_close (AsCache *cache); - -gboolean as_cache_insert (AsCache *cache, - AsComponent *cpt, - GError **error); - -gboolean as_cache_remove_by_data_id (AsCache *cache, - const gchar *cdid, - GError **error); - -GPtrArray *as_cache_get_components_all (AsCache *cache, - GError **error); -GPtrArray *as_cache_get_components_by_id (AsCache *cache, - const gchar *id, - GError **error); -AsComponent *as_cache_get_component_by_data_id (AsCache *cache, - const gchar *cdid, - GError **error); - -GPtrArray *as_cache_get_components_by_kind (AsCache *cache, - AsComponentKind kind, - GError **error); -GPtrArray *as_cache_get_components_by_provided_item (AsCache *cache, - AsProvidedKind kind, - const gchar *item, - GError **error); -GPtrArray *as_cache_get_components_by_categories (AsCache *cache, - gchar **categories, +AsCache *as_cache_new (void); + +const gchar *as_cache_get_locale (AsCache *cache); +void as_cache_set_locale (AsCache *cache, + const gchar *locale); + +void as_cache_set_locations (AsCache *cache, + const gchar *system_cache_dir, + const gchar *user_cache_dir); + +gboolean as_cache_get_prefer_os_metainfo (AsCache *cache); +void as_cache_set_prefer_os_metainfo (AsCache *cache, + gboolean prefer_os_metainfo); + +void as_cache_prune_data (AsCache *cache); + +void as_cache_clear (AsCache *cache); + +gboolean as_cache_set_contents_for_section (AsCache *cache, + AsComponentScope scope, + AsFormatStyle source_format_style, + gboolean is_os_data, + GPtrArray *cpts, + const gchar *cache_key, + gpointer refinefn_user_data, + GError **error); +gboolean as_cache_set_contents_for_path (AsCache *cache, + GPtrArray *cpts, + const gchar *filename, + gpointer refinefn_user_data, + GError **error); + +time_t as_cache_get_ctime (AsCache *cache, + AsComponentScope scope, + const gchar *cache_key, + AsCacheScope *cache_scope); + +void as_cache_load_section_for_key (AsCache *cache, + AsComponentScope scope, + AsFormatStyle source_format_style, + gboolean is_os_data, + const gchar *cache_key, + gboolean *is_outdated, + gpointer refinefn_user_data); +void as_cache_load_section_for_path (AsCache *cache, + const gchar *filename, + gboolean *is_outdated, + gpointer refinefn_user_data); + +void as_cache_mask_by_data_id (AsCache *cache, + const gchar *cdid); +gboolean as_cache_add_masking_components (AsCache *cache, + GPtrArray *cpts, + GError **error); + +void as_cache_set_refine_func (AsCache *cache, + AsCacheDataRefineFn func); + +GPtrArray *as_cache_get_components_all (AsCache *cache, GError **error); -GPtrArray *as_cache_get_components_by_launchable (AsCache *cache, - AsLaunchableKind kind, - const gchar *id, - GError **error); - -GPtrArray *as_cache_search (AsCache *cache, - gchar **terms, - gboolean sort, - GError **error); - -gboolean as_cache_has_component_id (AsCache *cache, - const gchar *id, - GError **error); -gssize as_cache_count_components (AsCache *cache, - GError **error); - -time_t as_cache_get_ctime (AsCache *cache); -gboolean as_cache_is_open (AsCache *cache); - -void as_cache_make_floating (AsCache *cache); -guint as_cache_unfloat (AsCache *cache, - GError **error); - -const gchar *as_cache_get_location (AsCache *cache); -void as_cache_set_location (AsCache *cache, - const gchar *location); - -void as_cache_set_refine_func (AsCache *cache, - GFunc func, - gpointer user_data); - -gboolean as_cache_get_nosync (AsCache *cache); -void as_cache_set_nosync (AsCache *cache, - gboolean nosync); - -gboolean as_cache_get_readonly (AsCache *cache); -void as_cache_set_readonly (AsCache *cache, - gboolean ro); + +GPtrArray *as_cache_get_components_by_id (AsCache *cache, + const gchar *id, + GError **error); + +GPtrArray *as_cache_get_components_by_extends (AsCache *cache, + const gchar *extends_id, + GError **error); + +GPtrArray *as_cache_get_components_by_kind (AsCache *cache, + AsComponentKind kind, + GError **error); + +GPtrArray *as_cache_get_components_by_provided_item (AsCache *cache, + AsProvidedKind kind, + const gchar *item, + GError **error); + +GPtrArray *as_cache_get_components_by_categories (AsCache *cache, + gchar **categories, + GError **error); + +GPtrArray *as_cache_get_components_by_launchable (AsCache *cache, + AsLaunchableKind kind, + const gchar *id, + GError **error); + +GPtrArray *as_cache_search (AsCache *cache, + gchar **terms, + gboolean sort, + GError **error); G_END_DECLS diff --git a/src/as-component.c b/src/as-component.c index 678aed4b7..14f3ea194 100644 --- a/src/as-component.c +++ b/src/as-component.c @@ -2381,8 +2381,12 @@ as_component_complete (AsComponent *cpt, gchar *scr_service_url, GPtrArray *icon { AsComponentPrivate *priv = GET_PRIVATE (cpt); + /* add search tokens */ + as_component_create_token_cache (cpt); + /* improve icon paths */ - as_component_refine_icons (cpt, icon_paths); + if (icon_paths != NULL) + as_component_refine_icons (cpt, icon_paths); /* "fake" a launchable entry for desktop-apps that failed to include one. This is used for legacy compatibility */ if ((priv->kind == AS_COMPONENT_KIND_DESKTOP_APP) && (priv->launchables->len <= 0)) { @@ -3798,9 +3802,7 @@ as_component_load_from_xml (AsComponent *cpt, AsContext *ctx, xmlNode *node, GEr if (content != NULL) as_component_set_compulsory_for_desktop (cpt, content); } else if (tag_id == AS_TAG_RELEASES) { - xmlNode *iter2; - - for (iter2 = iter->children; iter2 != NULL; iter2 = iter2->next) { + for (xmlNode *iter2 = iter->children; iter2 != NULL; iter2 = iter2->next) { if (iter2->type != XML_ELEMENT_NODE) continue; if (g_strcmp0 ((const gchar*) iter2->name, "release") == 0) { @@ -3862,6 +3864,18 @@ as_component_load_from_xml (AsComponent *cpt, AsContext *ctx, xmlNode *node, GEr as_component_set_origin (cpt, content); } else if (tag_id == AS_TAG_INTERNAL_BRANCH) { as_component_set_branch (cpt, content); + } else if (tag_id == AS_TAG_INTERNAL_TOKENS) { + for (xmlNode *iter2 = iter->children; iter2 != NULL; iter2 = iter2->next) { + AsTokenType *match_pval; + if (iter2->type != XML_ELEMENT_NODE) + continue; + + match_pval = g_new0 (AsTokenType, 1); + *match_pval = as_xml_get_prop_value_as_int (iter2, "score"); + g_hash_table_insert (priv->token_cache, + as_xml_get_node_value_refstr (iter2), + match_pval); + } } } else if (tag_id == AS_TAG_NAME_VARIANT_SUFFIX) { if (lang != NULL) @@ -4266,6 +4280,10 @@ as_component_to_xml_node (AsComponent *cpt, AsContext *ctx, xmlNode *root) /* internal information */ if (as_context_get_internal_mode (ctx)) { + GHashTableIter tc_iter; + gpointer tc_value, tc_key; + xmlNode *tc_node; + const gchar *origin = as_component_get_origin (cpt); if (priv->scope != AS_COMPONENT_SCOPE_UNKNOWN) as_xml_add_text_node (cnode, "__asi_scope", as_component_scope_to_string (priv->scope)); @@ -4273,6 +4291,25 @@ as_component_to_xml_node (AsComponent *cpt, AsContext *ctx, xmlNode *root) as_xml_add_text_node (cnode, "__asi_origin", origin); if (priv->branch != NULL) as_xml_add_text_node (cnode, "__asi_branch", priv->branch); + + tc_node = xmlNewChild (cnode, + NULL, + (xmlChar*) "__asi_tokens", + NULL); + + g_hash_table_iter_init (&tc_iter, priv->token_cache); + while (g_hash_table_iter_next (&tc_iter, &tc_key, &tc_value)) { + g_autofree gchar *tmp = NULL; + AsTokenType *score_val; + xmlNode *tct = as_xml_add_text_node (tc_node, + "t", (const gchar*) tc_key); + score_val = tc_value; + if (*score_val == 0) + continue; + + tmp = g_strdup_printf ("%" G_GUINT16_FORMAT, *score_val); + as_xml_add_text_prop (tct, "score", tmp); + } } return cnode; diff --git a/src/as-distro-extras.c b/src/as-distro-extras.c index 7a625621a..d05d48a9e 100644 --- a/src/as-distro-extras.c +++ b/src/as-distro-extras.c @@ -203,7 +203,7 @@ as_pool_check_file_newer_than_cache (AsPool *pool, GPtrArray *file_list) const gchar *fname = (const gchar*) g_ptr_array_index (file_list, i); if (stat (fname, &sb) < 0) continue; - if (sb.st_ctime > as_pool_get_system_cache_age (pool)) { + if (sb.st_ctime > as_pool_get_os_metadata_cache_age (pool)) { /* we need to update the cache */ return TRUE; } diff --git a/src/as-pool-private.h b/src/as-pool-private.h index e10f5fa82..852ceb67b 100644 --- a/src/as-pool-private.h +++ b/src/as-pool-private.h @@ -27,7 +27,7 @@ G_BEGIN_DECLS #pragma GCC visibility push(hidden) -time_t as_pool_get_system_cache_age (AsPool *pool); +time_t as_pool_get_os_metadata_cache_age (AsPool *pool); AS_INTERNAL_VISIBLE void as_cache_file_save (const gchar *fname, @@ -41,9 +41,15 @@ GPtrArray *as_cache_file_read (const gchar *fname, AS_INTERNAL_VISIBLE gboolean as_pool_refresh_system_cache (AsPool *pool, - gboolean force, - gboolean cleanup_old, - GError **error); + gboolean user, + gboolean force, + gboolean *caches_updated, + GError **error); + +AS_INTERNAL_VISIBLE +void as_pool_override_cache_locations (AsPool *pool, + const gchar *dir_sys, + const gchar *dir_user); #pragma GCC visibility pop G_END_DECLS diff --git a/src/as-pool.c b/src/as-pool.c index 72e3def29..0f998e3e9 100644 --- a/src/as-pool.c +++ b/src/as-pool.c @@ -65,28 +65,22 @@ typedef struct { - gchar *screenshot_service_url; - gchar *locale; - gchar *current_arch; - AsProfile *profile; + gchar *screenshot_service_url; + gchar *locale; + gchar *current_arch; + AsProfile *profile; - GPtrArray *xml_dirs; - GPtrArray *yaml_dirs; - GPtrArray *icon_dirs; + GHashTable *std_data_locations; + GHashTable *extra_data_locations; - AsCache *system_cache; - AsCache *cache; - gchar *cache_fname; - gchar *sys_cache_dir_system; - gchar *sys_cache_dir_user; + AsCache *cache; - gchar **term_greylist; + gchar **term_greylist; - AsPoolFlags flags; - AsCacheFlags cache_flags; - gboolean prefer_local_metainfo; + AsPoolFlags flags; + gboolean prefer_local_metainfo; - GMutex mutex; + GMutex mutex; } AsPoolPrivate; G_DEFINE_TYPE_WITH_PRIVATE (AsPool, as_pool, G_TYPE_OBJECT) @@ -116,9 +110,193 @@ static gchar *APPLICATIONS_DIR = "/usr/share/applications"; /* where metainfo files can be found */ static gchar *METAINFO_DIR = "/usr/share/metainfo"; -static void as_pool_add_metadata_location_internal (AsPool *pool, const gchar *directory, gboolean add_root); -static void as_pool_cache_refine_component_cb (gpointer data, gpointer user_data); -static void as_pool_cleanup_cache_dir (AsPool *pool, const gchar *cache_dir); +/* cache key used for local metainfo / desktop-entry data */ +static gchar *LOCAL_METAINFO_CACHE_KEY = "local-metainfo"; + +/* cache key used for AppStream collection metadata provided by the OS */ +static gchar *OS_COLLECTION_CACHE_KEY = "os-catalog"; + +static void as_pool_cache_refine_component_cb (AsComponent *cpt, gboolean is_serialization, gpointer user_data); + +typedef struct { + AsFormatKind format_kind; + GRefString *location; +} AsLocationEntry; + +static AsLocationEntry* +as_location_entry_new (AsFormatKind format_kind, + const gchar *location) +{ + AsLocationEntry *entry; + entry = g_new0 (AsLocationEntry, 1); + + entry->format_kind = format_kind; + as_ref_string_assign_safe (&entry->location, location); + + return entry; +} + +static void +as_location_entry_free (AsLocationEntry *entry) +{ + as_ref_string_release (entry->location); + g_free (entry); +} + +typedef struct { + AsPool *owner; + AsComponentScope scope; + AsFormatStyle format_style; + gboolean is_os_data; + GPtrArray *locations; + GPtrArray *icon_dirs; + GRefString *cache_key; +} AsLocationGroup; + +static AsLocationGroup* +as_location_group_new (AsPool *owner, + AsComponentScope scope, + AsFormatStyle format_style, + gboolean is_os_data, + const gchar *cache_key) +{ + AsLocationGroup *lgroup; + lgroup = g_new0 (AsLocationGroup, 1); + + lgroup->owner = owner; + lgroup->scope = scope; + lgroup->format_style = format_style; + lgroup->is_os_data = is_os_data; + + lgroup->locations = g_ptr_array_new_with_free_func ((GDestroyNotify) as_location_entry_free); + lgroup->icon_dirs = g_ptr_array_new_with_free_func ((GDestroyNotify) as_ref_string_release); + as_ref_string_assign_safe (&lgroup->cache_key, cache_key); + + return lgroup; +} + +static void +as_location_group_free (AsLocationGroup *lgroup) +{ + g_ptr_array_unref (lgroup->locations); + g_ptr_array_unref (lgroup->icon_dirs); + as_ref_string_release (lgroup->cache_key); + + g_free (lgroup); +} + +static void +as_location_group_add_dir (AsLocationGroup *lgroup, + const gchar *dir, + const gchar *icon_dir, + AsFormatKind format_kind) +{ + AsLocationEntry *entry; + g_return_if_fail (dir != NULL); + + entry = as_location_entry_new (format_kind, dir); + g_ptr_array_add (lgroup->locations, entry); + if (icon_dir != NULL) + g_ptr_array_add (lgroup->icon_dirs, + g_ref_string_new_intern (icon_dir)); +} + +G_DEFINE_AUTOPTR_CLEANUP_FUNC(AsLocationGroup, as_location_group_free) + +/** + * AsComponentRegistry: + * + * A helper object to refine partitions of + * AppStream metadata properly. + */ +typedef struct { + GHashTable *data_id_map; + GHashTable *id_map; +} AsComponentRegistry; + +static AsComponentRegistry* +as_component_registry_new (void) +{ + AsComponentRegistry *registry; + registry = g_new0 (AsComponentRegistry, 1); + + registry->data_id_map = g_hash_table_new_full (g_str_hash, g_str_equal, + NULL, g_object_unref); + registry->id_map = g_hash_table_new_full (g_str_hash, g_str_equal, + NULL, (GDestroyNotify) g_ptr_array_unref); + + return registry; +} + +static void +as_component_registry_free (AsComponentRegistry *registry) +{ + g_hash_table_unref (registry->data_id_map); + g_hash_table_unref (registry->id_map); + + g_free (registry); +} + +static void +as_component_registry_add (AsComponentRegistry *registry, AsComponent *cpt) +{ + GPtrArray *list; + + g_hash_table_insert (registry->data_id_map, + (gchar*) as_component_get_data_id (cpt), + g_object_ref (cpt)); + + list = g_hash_table_lookup (registry->id_map, as_component_get_id (cpt)); + if (list == NULL) { + list = g_ptr_array_new_with_free_func (g_object_unref); + g_hash_table_insert (registry->id_map, + (gchar*) as_component_get_id (cpt), + list); + } + g_ptr_array_add (list, + g_object_ref (cpt)); +} + +static AsComponent* +as_component_registry_lookup (AsComponentRegistry *registry, const gchar *data_id) +{ + return g_hash_table_lookup (registry->data_id_map, data_id); +} + +static void +as_component_registry_remove (AsComponentRegistry *registry, const gchar *data_id) +{ + g_hash_table_remove (registry->data_id_map, data_id); +} + +static gboolean +as_component_registry_has_cid (AsComponentRegistry *registry, const gchar *cid) +{ + return g_hash_table_contains (registry->id_map, cid); +} + +static GPtrArray* +as_component_registry_get_components_by_id (AsComponentRegistry *registry, const gchar *cid) +{ + return g_hash_table_lookup (registry->id_map, cid); +} + +static GPtrArray* +as_component_registry_get_contents (AsComponentRegistry *registry) +{ + GPtrArray *cpt_array; + GHashTableIter iter; + gpointer value; + + cpt_array = g_ptr_array_new_with_free_func (g_object_unref); + g_hash_table_iter_init (&iter, registry->data_id_map); + while (g_hash_table_iter_next (&iter, NULL, &value)) + g_ptr_array_add (cpt_array, g_object_ref (value)); + + return cpt_array; +} + +G_DEFINE_AUTOPTR_CLEANUP_FUNC(AsComponentRegistry, as_component_registry_free) /** * as_pool_init: @@ -138,9 +316,13 @@ as_pool_init (AsPool *pool) /* set active locale */ priv->locale = as_get_current_locale (); - priv->xml_dirs = g_ptr_array_new_with_free_func (g_free); - priv->yaml_dirs = g_ptr_array_new_with_free_func (g_free); - priv->icon_dirs = g_ptr_array_new_with_free_func (g_free); + /* well-known default metadata directories */ + priv->std_data_locations = g_hash_table_new_full (g_str_hash, g_str_equal, + g_free, (GDestroyNotify) as_location_group_free); + + /* user-defined collection metadata locations */ + priv->extra_data_locations = g_hash_table_new_full (g_str_hash, g_str_equal, + g_free, (GDestroyNotify) as_location_group_free); /* set the current architecture */ priv->current_arch = as_get_current_arch (); @@ -148,38 +330,11 @@ as_pool_init (AsPool *pool) /* set up our localized search-term greylist */ priv->term_greylist = g_strsplit (AS_SEARCH_GREYLIST_STR, ";", -1); - /* system-wide system data cache locations */ - priv->sys_cache_dir_system = g_strdup (AS_APPSTREAM_CACHE_PATH); - - /* per-user system data cache locations */ - cache_root = as_get_user_cache_dir (); - priv->sys_cache_dir_user = g_build_filename (cache_root, "system", NULL); - - if (as_utils_is_root ()) { - /* users umask shouldn't interfere with us creating new files when we are root */ - as_reset_umask (); - - /* ensure we never start gvfsd as root: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=852696 */ - g_setenv ("GIO_USE_VFS", "local", TRUE); - } - /* create caches */ - priv->system_cache = as_cache_new (); priv->cache = as_cache_new (); - /* system cache is always read-only */ - as_cache_set_readonly (priv->system_cache, TRUE); - /* set callback to refine components after deserialization */ - as_cache_set_refine_func (priv->cache, as_pool_cache_refine_component_cb, pool); - as_cache_set_refine_func (priv->system_cache, as_pool_cache_refine_component_cb, pool); - - /* open our session cache in temporary mode by default */ - priv->cache_fname = g_strdup (":temporary"); - if (!as_cache_open (priv->cache, priv->cache_fname, priv->locale, &tmp_error)) { - g_critical ("Unable to open temporary cache: %s", tmp_error->message); - g_clear_pointer (&tmp_error, g_error_free); - } + as_cache_set_refine_func (priv->cache, as_pool_cache_refine_component_cb); distro = as_distro_details_new (); priv->screenshot_service_url = as_distro_details_get_str (distro, "ScreenshotUrl"); @@ -187,15 +342,10 @@ as_pool_init (AsPool *pool) /* check whether we might want to prefer local metainfo files over remote data */ priv->prefer_local_metainfo = as_distro_details_get_bool (distro, "PreferLocalMetainfoData", FALSE); - /* set watched default directories for AppStream metadata */ - for (guint i = 0; AS_SYSTEM_COLLECTION_METADATA_PATHS[i] != NULL; i++) - as_pool_add_metadata_location_internal (pool, AS_SYSTEM_COLLECTION_METADATA_PATHS[i], FALSE); - /* set default pool flags */ - priv->flags = AS_POOL_FLAG_READ_COLLECTION | AS_POOL_FLAG_READ_METAINFO; - - /* set default cache flags */ - priv->cache_flags = AS_CACHE_FLAG_USE_SYSTEM | AS_CACHE_FLAG_USE_USER | AS_CACHE_FLAG_REFRESH_SYSTEM; + priv->flags = AS_POOL_FLAG_LOAD_OS_COLLECTION | + AS_POOL_FLAG_LOAD_OS_METAINFO | + AS_POOL_FLAG_LOAD_FLATPAK; } /** @@ -210,15 +360,10 @@ as_pool_finalize (GObject *object) g_mutex_lock (&priv->mutex); g_free (priv->screenshot_service_url); - g_ptr_array_unref (priv->xml_dirs); - g_ptr_array_unref (priv->yaml_dirs); - g_ptr_array_unref (priv->icon_dirs); + g_hash_table_unref (priv->std_data_locations); + g_hash_table_unref (priv->extra_data_locations); g_object_unref (priv->cache); - g_object_unref (priv->system_cache); - g_free (priv->cache_fname); - g_free (priv->sys_cache_dir_user); - g_free (priv->sys_cache_dir_system); g_free (priv->locale); g_free (priv->current_arch); @@ -243,115 +388,156 @@ as_pool_class_init (AsPoolClass *klass) } /** - * as_pool_can_query_system_cache: + * as_pool_add_collection_metadata_dir_internal: + * @pool: An instance of #AsPool. + * @lgroup: The location grouping to add to. + * @directory: An existing filesystem location. + * @add_root: Whether to add the root directory if necessary. * - * Check whether the system cache can be used for reading data. + * See %as_pool_add_metadata_location() */ -static inline gboolean -as_pool_can_query_system_cache (AsPool *pool) +static void +as_pool_add_collection_metadata_dir_internal (AsPool *pool, + AsLocationGroup *lgroup, + const gchar *directory, + gboolean add_root) { - AsPoolPrivate *priv = GET_PRIVATE (pool); - g_autoptr(GMutexLocker) locker = g_mutex_locker_new (&priv->mutex); - if (as_flags_contains (priv->cache_flags, AS_CACHE_FLAG_USE_SYSTEM)) - return as_cache_is_open (priv->system_cache); - return FALSE; -} + gboolean dir_added = FALSE; + g_autofree gchar *icon_dir = NULL; + gchar *path; -/** - * as_pool_get_component_by_data_id: - */ -static AsComponent* -as_pool_get_component_by_data_id (AsPool *pool, const gchar *cdid, GError **error) -{ - AsPoolPrivate *priv = GET_PRIVATE (pool); - GError *tmp_error = NULL; - AsComponent *cpt; - cpt = as_cache_get_component_by_data_id (priv->cache, cdid, &tmp_error); - if (cpt != NULL) - return cpt; - if (tmp_error != NULL) { - g_propagate_error (error, tmp_error); - return NULL; + if (!g_file_test (directory, G_FILE_TEST_IS_DIR)) { + g_debug ("Not adding metadata location '%s': Not a directory, or does not exist.", + directory); + return; } - /* check system cache last */ - if (as_pool_can_query_system_cache (pool)) - return as_cache_get_component_by_data_id (priv->system_cache, cdid, &tmp_error); - else - return NULL; -} + /* icons */ + icon_dir = g_build_filename (directory, "icons", NULL); + if (!g_file_test (icon_dir, G_FILE_TEST_IS_DIR)) + g_free (g_steal_pointer (&icon_dir)); -/** - * as_pool_remove_by_data_id: - */ -static gboolean -as_pool_remove_by_data_id (AsPool *pool, const gchar *cdid, GError **error) -{ - AsPoolPrivate *priv = GET_PRIVATE (pool); - GError *tmp_error = NULL; + /* metadata locations */ + path = g_build_filename (directory, "xml", NULL); + if (g_file_test (path, G_FILE_TEST_IS_DIR)) { + as_location_group_add_dir (lgroup, + path, + icon_dir, + AS_FORMAT_KIND_XML); + dir_added = TRUE; + } + g_free (path); - if (as_pool_can_query_system_cache (pool)) { - as_cache_remove_by_data_id (priv->system_cache, cdid, &tmp_error); - if (tmp_error != NULL) { - g_propagate_error (error, tmp_error); - return FALSE; - } + path = g_build_filename (directory, "xmls", NULL); + if (g_file_test (path, G_FILE_TEST_IS_DIR)) { + as_location_group_add_dir (lgroup, + path, + icon_dir, + AS_FORMAT_KIND_XML); + dir_added = TRUE; } - return as_cache_remove_by_data_id (priv->cache, cdid, error); -} + g_free (path); -/** - * as_pool_insert: - */ -static gboolean -as_pool_insert (AsPool *pool, AsComponent *cpt, GError **error) -{ - AsPoolPrivate *priv = GET_PRIVATE (pool); - GError *tmp_error = NULL; + path = g_build_filename (directory, "yaml", NULL); + if (g_file_test (path, G_FILE_TEST_IS_DIR)) { + as_location_group_add_dir (lgroup, + path, + icon_dir, + AS_FORMAT_KIND_YAML); + dir_added = TRUE; + } + g_free (path); - /* if we have a system cache, ensure the component is "removed" (masked) there, - * and re-added then to the current session cache */ - if (as_pool_can_query_system_cache (pool)) { - as_cache_remove_by_data_id (priv->system_cache, - as_component_get_data_id (cpt), - &tmp_error); - if (tmp_error != NULL) { - g_propagate_error (error, tmp_error); - return FALSE; - } + if ((add_root) && (!dir_added)) { + /* we didn't find metadata-specific directories, so let's watch to root path for both YAML and XML */ + as_location_group_add_dir (lgroup, + directory, + icon_dir, + AS_FORMAT_KIND_XML); + as_location_group_add_dir (lgroup, + directory, + icon_dir, + AS_FORMAT_KIND_YAML); + g_debug ("Added %s to YAML and XML metadata watch locations.", directory); } - return as_cache_insert (priv->cache, cpt, error); } /** - * as_pool_has_component_id: + * as_pool_detect_std_metadata_dirs: + * + * Find common AppStream metadata directories. */ -static gboolean -as_pool_has_component_id (AsPool *pool, const gchar *cid, GError **error) +static void +as_pool_detect_std_metadata_dirs (AsPool *pool, gboolean system_data_only) { AsPoolPrivate *priv = GET_PRIVATE (pool); - GError *tmp_error = NULL; - gboolean ret; + AsLocationGroup *lgroup_coll; + AsLocationGroup *lgroup_metainfo; + g_autoptr(GMutexLocker) locker = g_mutex_locker_new (&priv->mutex); - ret = as_cache_has_component_id (priv->cache, cid, &tmp_error); - if (tmp_error != NULL) { - g_propagate_error (error, tmp_error); - return FALSE; + /* clear existing entries */ + g_hash_table_remove_all (priv->std_data_locations); + + /* create location groups and register them */ + lgroup_coll = as_location_group_new (pool, + AS_COMPONENT_SCOPE_SYSTEM, + AS_FORMAT_STYLE_COLLECTION, + TRUE, /* is OS data */ + OS_COLLECTION_CACHE_KEY); + g_hash_table_insert (priv->std_data_locations, + g_strdup (lgroup_coll->cache_key), + lgroup_coll); + lgroup_metainfo = as_location_group_new (pool, + AS_COMPONENT_SCOPE_SYSTEM, + AS_FORMAT_STYLE_METAINFO, + TRUE, /* is OS data */ + LOCAL_METAINFO_CACHE_KEY); + g_hash_table_insert (priv->std_data_locations, + g_strdup (lgroup_metainfo->cache_key), + lgroup_metainfo); + + /* we always need to look at both if either Metainfo loading or desktop-entry loading is enabled. + * the fine sorting can happen later. */ + if (as_flags_contains (priv->flags, AS_POOL_FLAG_LOAD_OS_DESKTOP_FILES) || + as_flags_contains (priv->flags, AS_POOL_FLAG_LOAD_OS_METAINFO)) { + /* desktop-entry */ + if (g_file_test (APPLICATIONS_DIR, G_FILE_TEST_IS_DIR)) { + as_location_group_add_dir (lgroup_metainfo, + APPLICATIONS_DIR, + NULL, /* no icon dir */ + AS_FORMAT_KIND_DESKTOP_ENTRY); + } else { + g_debug ("System applications desktop-entry directory was not found!"); + } + + /* metainfo files */ + if (g_file_test (METAINFO_DIR, G_FILE_TEST_IS_DIR)) { + as_location_group_add_dir (lgroup_metainfo, + METAINFO_DIR, + NULL, /* no icon dir */ + AS_FORMAT_KIND_XML); + } else { + g_debug ("System installed MetaInfo directory was not found!"); + } } - if (ret) - return ret; - /* check system cache last */ - if (as_pool_can_query_system_cache (pool)) - return as_cache_has_component_id (priv->system_cache, cid, &tmp_error); - else - return FALSE; + /* add collection XML directories for the OS */ + /* check if we are permitted to load this */ + if (as_flags_contains (priv->flags, AS_POOL_FLAG_LOAD_OS_COLLECTION)) { + for (guint i = 0; AS_SYSTEM_COLLECTION_METADATA_PATHS[i] != NULL; i++) { + as_pool_add_collection_metadata_dir_internal (pool, + lgroup_coll, + AS_SYSTEM_COLLECTION_METADATA_PATHS[i], + FALSE); + } + } } /** * as_pool_add_component_internal: * @pool: An instance of #AsPool + * @registry: Hash table for the results / known components * @cpt: The #AsComponent to add to the pool. * @pedantic_noadd: If %TRUE, always emit an error if component couldn't be added. * @error: A #GError or %NULL @@ -359,16 +545,19 @@ as_pool_has_component_id (AsPool *pool, const gchar *cid, GError **error) * Internal. */ static gboolean -as_pool_add_component_internal (AsPool *pool, AsComponent *cpt, gboolean pedantic_noadd, GError **error) +as_pool_add_component_internal (AsPool *pool, + AsComponentRegistry *registry, + AsComponent *cpt, + gboolean pedantic_noadd, + GError **error) { AsPoolPrivate *priv = GET_PRIVATE (pool); const gchar *cdid = NULL; - g_autoptr(AsComponent) existing_cpt = NULL; + AsComponent *existing_cpt; gint pool_priority; AsOriginKind new_cpt_orig_kind; AsOriginKind existing_cpt_orig_kind; AsMergeKind new_cpt_merge_kind; - GError *tmp_error = NULL; cdid = as_component_get_data_id (cpt); if (as_component_is_ignored (cpt)) { @@ -380,27 +569,16 @@ as_pool_add_component_internal (AsPool *pool, AsComponent *cpt, gboolean pedanti return FALSE; } - existing_cpt = as_pool_get_component_by_data_id (pool, cdid, &tmp_error); - if (tmp_error != NULL) { - g_propagate_error (error, tmp_error); - return FALSE; - } - + existing_cpt = as_component_registry_lookup (registry, cdid); if (as_component_get_origin_kind (cpt) == AS_ORIGIN_KIND_DESKTOP_ENTRY) { - g_autofree gchar *tmp_cdid = NULL; - /* .desktop entries might map to existing metadata data with or without .desktop suffix, we need to check for that. * (the .desktop suffix is optional for desktop-application metainfo files, and the desktop-entry parser will automatically * omit it if the desktop-entry-id is following the reverse DNS scheme) */ if (existing_cpt == NULL) { + g_autofree gchar *tmp_cdid = NULL; tmp_cdid = g_strdup_printf ("%s.desktop", cdid); - - existing_cpt = as_pool_get_component_by_data_id (pool, tmp_cdid, &tmp_error); - if (tmp_error != NULL) { - g_propagate_error (error, tmp_error); - return FALSE; - } + existing_cpt = as_component_registry_lookup (registry, tmp_cdid); } if (existing_cpt != NULL) { @@ -416,23 +594,21 @@ as_pool_add_component_internal (AsPool *pool, AsComponent *cpt, gboolean pedanti /* perform metadata merges if necessary */ new_cpt_merge_kind = as_component_get_merge_kind (cpt); if (new_cpt_merge_kind != AS_MERGE_KIND_NONE) { - g_autoptr(GPtrArray) matches = NULL; - guint i; + GPtrArray *matches; /* we merge the data into all components with matching IDs at time */ - matches = as_pool_get_components_by_id (pool, - as_component_get_id (cpt)); - for (i = 0; i < matches->len; i++) { + matches = as_component_registry_get_components_by_id (registry, + as_component_get_id (cpt)); + if (matches == NULL) + return TRUE; + + for (guint i = 0; i < matches->len; i++) { AsComponent *match = AS_COMPONENT (g_ptr_array_index (matches, i)); if (new_cpt_merge_kind == AS_MERGE_KIND_REMOVE_COMPONENT) { /* remove matching component from pool if its priority is lower */ if (as_component_get_priority (match) < as_component_get_priority (cpt)) { const gchar *match_cdid = as_component_get_data_id (match); - as_pool_remove_by_data_id (pool, match_cdid, &tmp_error); - if (tmp_error != NULL) { - g_propagate_error (error, tmp_error); - return FALSE; - } + as_component_registry_remove (registry, match_cdid); g_debug ("Removed via merge component: %s", match_cdid); } } else { @@ -444,16 +620,14 @@ as_pool_add_component_internal (AsPool *pool, AsComponent *cpt, gboolean pedanti } if (existing_cpt == NULL) { - if (!as_pool_insert (pool, cpt, error)) - return FALSE; + as_component_registry_add (registry, cpt); return TRUE; } /* safety check so we don't ignore a good component because we added a bad one first */ if (!as_component_is_valid (existing_cpt)) { g_debug ("Replacing invalid component '%s' with new one.", cdid); - if (!as_pool_insert (pool, cpt, error)) - return FALSE; + as_component_registry_add (registry, cpt); return TRUE; } @@ -468,8 +642,7 @@ as_pool_add_component_internal (AsPool *pool, AsComponent *cpt, gboolean pedanti existing_cpt, AS_MERGE_KIND_APPEND); - if (!as_pool_insert (pool, cpt, error)) - return FALSE; + as_component_registry_add (registry, cpt); g_debug ("Replaced '%s' with data from metainfo and desktop-entry file.", cdid); return TRUE; } else { @@ -502,8 +675,7 @@ as_pool_add_component_internal (AsPool *pool, AsComponent *cpt, gboolean pedanti as_component_set_pkgnames (cpt, as_component_get_pkgnames (existing_cpt)); as_component_set_bundles_array (cpt, as_component_get_bundles (existing_cpt)); - if (!as_pool_insert (pool, cpt, error)) - return FALSE; + as_component_registry_add (registry, cpt); g_debug ("Replaced '%s' with data from metainfo file.", cdid); return TRUE; } @@ -512,8 +684,7 @@ as_pool_add_component_internal (AsPool *pool, AsComponent *cpt, gboolean pedanti * with data of higher priority, or if we have an actual error in the metadata */ pool_priority = as_component_get_priority (existing_cpt); if (pool_priority < as_component_get_priority (cpt)) { - if (!as_pool_insert (pool, cpt, error)) - return FALSE; + as_component_registry_add (registry, cpt); g_debug ("Replaced '%s' with data of higher priority.", cdid); } else { /* bundles are treated specially here */ @@ -534,8 +705,7 @@ as_pool_add_component_internal (AsPool *pool, AsComponent *cpt, gboolean pedanti earch = as_component_get_architecture (existing_cpt); if (earch != NULL) { if (as_arch_compatible (earch, priv->current_arch)) { - if (!as_pool_insert (pool, cpt, error)) - return FALSE; + as_component_registry_add (registry, cpt); g_debug ("Preferred component for native architecture for %s (was %s)", cdid, earch); return TRUE; } else { @@ -565,6 +735,28 @@ as_pool_add_component_internal (AsPool *pool, AsComponent *cpt, gboolean pedanti return TRUE; } +/** + * as_pool_add_components: + * @pool: An instance of #AsPool + * @cpts: (element-type AsComponent): Array of components to add to the pool. + * @error: A #GError or %NULL + * + * Register a set of components with the pool temporarily. + * Data from components added like this will not be cached. + * + * Returns: %TRUE if the new components were successfully added to the pool. + * + * Since: 0.14.8 + */ +gboolean +as_pool_add_components (AsPool *pool, GPtrArray *cpts, GError **error) +{ + AsPoolPrivate *priv = GET_PRIVATE (pool); + return as_cache_add_masking_components (priv->cache, + cpts, + error); +} + /** * as_pool_add_component: * @pool: An instance of #AsPool @@ -574,18 +766,23 @@ as_pool_add_component_internal (AsPool *pool, AsComponent *cpt, gboolean pedanti * Register a new component in the AppStream metadata pool. * * Returns: %TRUE if the new component was successfully added to the pool. + * + * Deprecated: 0.14.7: This function is very inefficient. Collect all the components you need + * to add, and then register them with %as_pool_add_components in one go. */ gboolean as_pool_add_component (AsPool *pool, AsComponent *cpt, GError **error) { - return as_pool_add_component_internal (pool, cpt, TRUE, error); + g_autoptr(GPtrArray) array = g_ptr_array_new (); + g_ptr_array_add (array, cpt); + return as_pool_add_components (pool, array, error); } /** * as_pool_clear2: * @pool: An #AsPool. * - * Remove all metadata from the pool and clear caches. + * Remove all metadata from the pool. */ gboolean as_pool_clear2 (AsPool *pool, GError **error) @@ -593,45 +790,17 @@ as_pool_clear2 (AsPool *pool, GError **error) AsPoolPrivate *priv = GET_PRIVATE (pool); g_autoptr(GMutexLocker) locker = g_mutex_locker_new (&priv->mutex); - /* close system cache so it won't be used anymore - * (will be loaded explicitly again later, when needed) */ - as_cache_close (priv->system_cache); + as_cache_clear (priv->cache); + as_cache_set_locale (priv->cache, priv->locale); - /* if we were just created, we may be able to reuse the current temporary cache, - * instead of creating a new one (which is a bit wasteful). - * Reuse requires a temporary cache with no elements. */ - if ((g_strcmp0 (priv->cache_fname, ":temporary") == 0) && - (as_cache_count_components (priv->cache, NULL) == 0)) { - /* we can reuse the user cache */ - - g_debug ("Not clearing user cache: The cache was already empty."); - return TRUE; - } else { - /* it looks like we can not reuse the old cache, so now we need to clear - * the cache for real by deleting the old one and creating a new one */ - - g_debug ("Clearing user cache."); - as_cache_close (priv->cache); - if (g_file_test (priv->cache_fname, G_FILE_TEST_EXISTS)) { - if (g_remove (priv->cache_fname) != 0) { - g_set_error_literal (error, - AS_POOL_ERROR, - AS_POOL_ERROR_OLD_CACHE, - _("Unable to remove old cache.")); - return FALSE; - } - } - - /* reopen the session cache as a new, pristine one */ - return as_cache_open (priv->cache, priv->cache_fname, priv->locale, error); - } + return TRUE; } /** * as_pool_clear: * @pool: An #AsPool. * - * Remove all metadat from the pool. + * Remove all metadata from the pool. */ void as_pool_clear (AsPool *pool) @@ -646,107 +815,17 @@ as_pool_clear (AsPool *pool) } } -/** - * as_pool_ctime_newer: - * - * Returns: %TRUE if ctime of file is newer than the cached time. - */ -static gboolean -as_pool_ctime_newer (AsPool *pool, const gchar *dir, AsCache *cache) -{ - struct stat sb; - time_t cache_ctime; - - if (stat (dir, &sb) < 0) - return FALSE; - - cache_ctime = as_cache_get_ctime (cache); - if (sb.st_ctime > cache_ctime) - return TRUE; - - return FALSE; -} - -/** - * as_path_is_system_metadata_location: - */ -static gboolean -as_path_is_system_metadata_location (const gchar *dir) -{ - /* we can't just do a "/home/" prefix check here, as e.g. Flatpak data may also be - * in system directories, and not every instance of an AppStream-using app will have - * these included, which would mess up cross-app cache sharing. - * In addition, some cliants may have multiple AsPool instance, further complicating - * this issue. */ - for (gint i = 0; AS_SYSTEM_COLLECTION_METADATA_PATHS[i] != NULL; i++) - if (g_str_has_prefix (dir, AS_SYSTEM_COLLECTION_METADATA_PATHS[i])) - return TRUE; - return FALSE; -} - -/** - * as_pool_has_system_metadata_paths: - */ -static gboolean -as_pool_has_system_metadata_paths (AsPool *pool) -{ - AsPoolPrivate *priv = GET_PRIVATE (pool); - guint i; - g_autoptr(GMutexLocker) locker = g_mutex_locker_new (&priv->mutex); - - for (i = 0; i < priv->xml_dirs->len; i++) { - const gchar *dir = (const gchar*) g_ptr_array_index (priv->xml_dirs, i); - if (as_path_is_system_metadata_location (dir)) - return TRUE; - } - for (i = 0; i < priv->yaml_dirs->len; i++) { - const gchar *dir = (const gchar*) g_ptr_array_index (priv->yaml_dirs, i); - if (as_path_is_system_metadata_location (dir)) - return TRUE; - } - - return FALSE; -} - -/** - * as_pool_appstream_data_changed: - */ -static gboolean -as_pool_metadata_changed (AsPool *pool, AsCache *cache, gboolean system_only) -{ - AsPoolPrivate *priv = GET_PRIVATE (pool); - g_autoptr(GMutexLocker) locker = g_mutex_locker_new (&priv->mutex); - - /* if the cache does not exist, we always need to recreate it */ - if (!g_file_test (as_cache_get_location (cache), G_FILE_TEST_EXISTS)) - return TRUE; - - /* compare file times */ - for (guint i = 0; i < priv->xml_dirs->len; i++) { - const gchar *dir = (const gchar*) g_ptr_array_index (priv->xml_dirs, i); - if (system_only && !as_path_is_system_metadata_location (dir)) - continue; - if (as_pool_ctime_newer (pool, dir, cache)) - return TRUE; - } - for (guint i = 0; i < priv->yaml_dirs->len; i++) { - const gchar *dir = (const gchar*) g_ptr_array_index (priv->yaml_dirs, i); - if (system_only && !as_path_is_system_metadata_location (dir)) - continue; - if (as_pool_ctime_newer (pool, dir, cache)) - return TRUE; - } - - return FALSE; -} - /** * as_pool_load_collection_data: * - * Load fresh metadata from AppStream collection data directories. + * Load metadata from AppStream collection data directories, + * which are usually provided by some kind of software repository. */ static gboolean -as_pool_load_collection_data (AsPool *pool, gboolean refresh, GError **error) +as_pool_load_collection_data (AsPool *pool, + AsComponentRegistry *registry, + AsLocationGroup *lgroup, + GError **error) { AsPoolPrivate *priv = GET_PRIVATE (pool); GPtrArray *cpts; @@ -757,113 +836,17 @@ as_pool_load_collection_data (AsPool *pool, gboolean refresh, GError **error) g_autoptr(GError) tmp_error = NULL; g_autoptr(GMutexLocker) locker = NULL; g_autoptr(AsProfileTask) ptask = NULL; - gboolean system_cache_used = FALSE; - /* see if we can use the system caches */ - if (!refresh && as_pool_has_system_metadata_paths (pool)) { - g_autofree gchar *cache_fname = NULL; - gboolean use_user_cache = FALSE; - gboolean cache_stale = TRUE; - - g_mutex_lock (&priv->mutex); - cache_fname = g_strdup_printf ("%s/%s.cache", priv->sys_cache_dir_system, priv->locale); - as_cache_set_location (priv->system_cache, cache_fname); - g_mutex_unlock (&priv->mutex); - if (!as_pool_metadata_changed (pool, priv->system_cache, TRUE)) { - g_debug ("Shared metadata cache of system data seems up to date."); - cache_stale = FALSE; - } else { - g_mutex_lock (&priv->mutex); - g_free (cache_fname); - cache_fname = g_strdup_printf ("%s/%s.cache", priv->sys_cache_dir_user, priv->locale); - as_cache_set_location (priv->system_cache, cache_fname); - g_mutex_unlock (&priv->mutex); - - use_user_cache = TRUE; - cache_stale = as_pool_metadata_changed (pool, priv->system_cache, TRUE); - if (cache_stale) - g_debug ("User metadata cache of system data is stale, may try to recreate it."); - else - g_debug ("User metadata cache of system data seems up to date."); - } - - g_mutex_lock (&priv->mutex); - if (as_flags_contains (priv->cache_flags, AS_CACHE_FLAG_USE_SYSTEM)) { - g_mutex_unlock (&priv->mutex); - - as_cache_close (priv->system_cache); - if (cache_stale) { - if (as_flags_contains (priv->cache_flags, AS_CACHE_FLAG_REFRESH_SYSTEM)) { - g_autoptr(AsPool) refresh_pool = as_pool_new (); - g_debug ("System-wide metadata cache is stale, will refresh it now."); - ret = as_pool_refresh_system_cache (refresh_pool, - TRUE, /* user */ - FALSE, /* force */ - &tmp_error); - if (tmp_error != NULL) { - if (ret) - g_warning ("System cache issue: %s", tmp_error->message); - else - g_warning ("Unable to refresh system cache: %s", tmp_error->message); - g_clear_pointer (&tmp_error, g_error_free); - } - if (ret) { - /* the cache should exist now, ready to be loaded */ - g_mutex_lock (&priv->mutex); - if (as_cache_open2 (priv->system_cache, priv->locale, &tmp_error)) { - system_cache_used = TRUE; - } else { - g_warning ("Unable to load newly generated system cache: %s", tmp_error->message); - g_clear_pointer (&tmp_error, g_error_free); - system_cache_used = FALSE; - } - g_mutex_unlock (&priv->mutex); - } - } else { - g_debug ("System-wide metadata cache is stale, but refresh was prohibited."); - system_cache_used = FALSE; - } - } else { - g_debug ("Using system cache data %s.", use_user_cache? "from user cache" : "from shared cache"); - - g_mutex_lock (&priv->mutex); - if (as_cache_open2 (priv->system_cache, priv->locale, &tmp_error)) { - system_cache_used = TRUE; - } else { - /* if we can't open the system cache for whatever reason, we complain but - * silently fall back to reading all data again */ - g_warning ("Unable to load system cache: %s", tmp_error->message); - g_clear_pointer (&tmp_error, g_error_free); - system_cache_used = FALSE; - } - g_mutex_unlock (&priv->mutex); - - /* try to clean up old caches for the user, in case the system cache is up to date - * and we are using it instead */ - if (!use_user_cache) - as_pool_cleanup_cache_dir (pool, priv->sys_cache_dir_user); - } - - } else { - g_mutex_unlock (&priv->mutex); - g_debug ("Not using system cache."); - as_cache_close (priv->system_cache); - system_cache_used = FALSE; - } - - } else { - if (!refresh) - g_debug ("No system collection metadata paths selected, can not use system cache."); - } + /* do nothing if the group has the wrong format */ + if (lgroup->format_style != AS_FORMAT_STYLE_COLLECTION) + return TRUE; ptask = as_profile_start_literal (priv->profile, "AsPool:load_collection_data"); /* prepare metadata parser */ metad = as_metadata_new (); as_metadata_set_format_style (metad, AS_FORMAT_STYLE_COLLECTION); - g_mutex_lock (&priv->mutex); as_metadata_set_locale (metad, priv->locale); - g_mutex_unlock (&priv->mutex); /* find AppStream metadata */ ret = TRUE; @@ -872,19 +855,18 @@ as_pool_load_collection_data (AsPool *pool, gboolean refresh, GError **error) /* protect access to directory lists */ locker = g_mutex_locker_new (&priv->mutex); - /* find XML data */ - for (guint i = 0; i < priv->xml_dirs->len; i++) { - const gchar *xml_path = (const gchar *) g_ptr_array_index (priv->xml_dirs, i); - if (system_cache_used && as_path_is_system_metadata_location (xml_path)) { - g_debug ("Skipped XML path '%s' for session cache: Already considered for system cache.", xml_path); + for (guint i = 0; i < lgroup->locations->len; i++) { + AsLocationEntry *lentry = g_ptr_array_index (lgroup->locations, i); + + /* skip things we can't handle */ + if (lentry->format_kind == AS_FORMAT_KIND_DESKTOP_ENTRY) continue; - } - if (g_file_test (xml_path, G_FILE_TEST_IS_DIR)) { + /* find XML data */ + if (lentry->format_kind == AS_FORMAT_KIND_XML) { g_autoptr(GPtrArray) xmls = NULL; - - g_debug ("Searching for data in: %s", xml_path); - xmls = as_utils_find_files_matching (xml_path, "*.xml*", FALSE, NULL); + g_debug ("Searching for data in: %s", lentry->location); + xmls = as_utils_find_files_matching (lentry->location, "*.xml*", FALSE, NULL); if (xmls != NULL) { for (guint j = 0; j < xmls->len; j++) { const gchar *val; @@ -894,21 +876,13 @@ as_pool_load_collection_data (AsPool *pool, gboolean refresh, GError **error) } } } - } - /* find YAML metadata */ - for (guint i = 0; i < priv->yaml_dirs->len; i++) { - const gchar *yaml_path = (const gchar *) g_ptr_array_index (priv->yaml_dirs, i); - if (system_cache_used && as_path_is_system_metadata_location (yaml_path)) { - g_debug ("Skipped YAML path '%s' for session cache: Already considered for system cache.", yaml_path); - continue; - } - - if (g_file_test (yaml_path, G_FILE_TEST_IS_DIR)) { + /* find YAML metadata */ + if (lentry->format_kind == AS_FORMAT_KIND_YAML) { g_autoptr(GPtrArray) yamls = NULL; - g_debug ("Searching for data in: %s", yaml_path); - yamls = as_utils_find_files_matching (yaml_path, "*.yml*", FALSE, NULL); + g_debug ("Searching for data in: %s", lentry->location); + yamls = as_utils_find_files_matching (lentry->location, "*.yml*", FALSE, NULL); if (yamls != NULL) { for (guint j = 0; j < yamls->len; j++) { const gchar *val; @@ -920,8 +894,6 @@ as_pool_load_collection_data (AsPool *pool, gboolean refresh, GError **error) } } - /* no need for further locking after this point, AsCache is threadsafe */ - g_clear_pointer (&locker, g_mutex_locker_free); /* parse the found data */ for (guint i = 0; i < mdata_files->len; i++) { @@ -968,8 +940,7 @@ as_pool_load_collection_data (AsPool *pool, gboolean refresh, GError **error) for (guint i = 0; i < cpts->len; i++) { AsComponent *cpt = AS_COMPONENT (g_ptr_array_index (cpts, i)); - /* TODO: We support only system components at time */ - as_component_set_scope (cpt, AS_COMPONENT_SCOPE_SYSTEM); + as_component_set_scope (cpt, lgroup->scope); /* deal with merge-components later */ if (as_component_get_merge_kind (cpt) != AS_MERGE_KIND_NONE) { @@ -977,7 +948,7 @@ as_pool_load_collection_data (AsPool *pool, gboolean refresh, GError **error) continue; } - as_pool_add_component_internal (pool, cpt, TRUE, &tmp_error); + as_pool_add_component_internal (pool, registry, cpt, TRUE, &tmp_error); if (tmp_error != NULL) { g_debug ("Metadata ignored: %s", tmp_error->message); g_clear_pointer (&tmp_error, g_error_free); @@ -989,7 +960,7 @@ as_pool_load_collection_data (AsPool *pool, gboolean refresh, GError **error) for (guint i = 0; i < merge_cpts->len; i++) { AsComponent *mcpt = AS_COMPONENT (g_ptr_array_index (merge_cpts, i)); - as_pool_add_component_internal (pool, mcpt, TRUE, &tmp_error); + as_pool_add_component_internal (pool, registry, mcpt, TRUE, &tmp_error); if (tmp_error != NULL) { g_debug ("Merge component ignored: %s", tmp_error->message); g_clear_pointer (&tmp_error, g_error_free); @@ -1000,20 +971,49 @@ as_pool_load_collection_data (AsPool *pool, gboolean refresh, GError **error) } /** - * as_pool_get_desktop_entries_table: + * as_pool_cache_refine_component_cb: * - * Load fresh metadata from .desktop files. + * Callback function run on components before they are + * (de)serialized. + */ +static void +as_pool_cache_refine_component_cb (AsComponent *cpt, gboolean is_serialization, gpointer user_data) +{ + AsLocationGroup *lgroup = user_data; + AsPool *pool = lgroup->owner; + AsPoolPrivate *priv = GET_PRIVATE (pool); + g_autoptr(GMutexLocker) locker = g_mutex_locker_new (&priv->mutex); + + + if (is_serialization) { + /* we refine everything except for the icon paths, as that is expensive on many components, + * and needs to be done partially again on deserialization anyway */ + /* FIXME: We *do* resolve icon paths here, as not doing so currently causes issues for some apps. + * There's a test case for this, so we can address the issue later. */ + as_component_complete (cpt, + priv->screenshot_service_url, + lgroup->icon_dirs); + } else { + /* add additional data to the component, e.g. external screenshots. Also refines + * the component's icon paths */ + as_component_complete (cpt, + priv->screenshot_service_url, + lgroup->icon_dirs); + } +} + +/** + * as_pool_update_desktop_entries_table: * - * Returns: (transfer full): a hash map of #AsComponent instances. + * Load metadata from desktop-entry files. */ -static GHashTable* -as_pool_get_desktop_entries_table (AsPool *pool) +static void +as_pool_update_desktop_entries_table (AsPool *pool, GHashTable *de_cpt_table, const gchar *apps_dir) { AsPoolPrivate *priv = GET_PRIVATE (pool); g_autoptr(AsMetadata) metad = NULL; g_autoptr(GPtrArray) de_files = NULL; g_autoptr(AsProfileTask) ptask = NULL; - GHashTable *de_cpt_table = NULL; GError *error = NULL; ptask = as_profile_start_literal (priv->profile, "AsPool:get_desktop_entries_table"); @@ -1024,17 +1024,12 @@ as_pool_get_desktop_entries_table (AsPool *pool) as_metadata_set_locale (metad, priv->locale); g_mutex_unlock (&priv->mutex); - de_cpt_table = g_hash_table_new_full (g_str_hash, - g_str_equal, - g_free, - (GDestroyNotify) g_object_unref); - /* find .desktop files */ - g_debug ("Searching for data in: %s", APPLICATIONS_DIR); - de_files = as_utils_find_files_matching (APPLICATIONS_DIR, "*.desktop", FALSE, NULL); + g_debug ("Searching for data in: %s", apps_dir); + de_files = as_utils_find_files_matching (apps_dir, "*.desktop", FALSE, NULL); if (de_files == NULL) { - g_debug ("Unable find .desktop files."); - return de_cpt_table; + g_debug ("Unable find desktop-entry files in '%s'.", apps_dir); + return; } /* parse the found data */ @@ -1072,17 +1067,20 @@ as_pool_get_desktop_entries_table (AsPool *pool) g_object_ref (cpt)); } } - - return de_cpt_table; } /** * as_pool_load_metainfo_data: * - * Load fresh metadata from metainfo files. + * Load metadata from locally installed metainfo files. */ static void -as_pool_load_metainfo_data (AsPool *pool, GHashTable *desktop_entry_cpts) +as_pool_load_metainfo_data (AsPool *pool, + AsComponentRegistry *registry, + GHashTable *desktop_entry_cpts, + AsComponentScope scope, + const gchar *metainfo_dir, + const gchar *cache_key) { AsPoolPrivate *priv = GET_PRIVATE (pool); g_autoptr(AsMetadata) metad = NULL; @@ -1090,17 +1088,15 @@ as_pool_load_metainfo_data (AsPool *pool, GHashTable *desktop_entry_cpts) g_autoptr(AsProfileTask) ptask = NULL; GError *error = NULL; - ptask = as_profile_start_literal (priv->profile, "AsPool:load_metainfo_data"); + ptask = as_profile_start (priv->profile, "AsPool:load_metainfo_data:%s", cache_key); /* prepare metadata parser */ metad = as_metadata_new (); - g_mutex_lock (&priv->mutex); as_metadata_set_locale (metad, priv->locale); - g_mutex_unlock (&priv->mutex); /* find metainfo files */ - g_debug ("Searching for data in: %s", METAINFO_DIR); - mi_files = as_utils_find_files_matching (METAINFO_DIR, "*.xml", FALSE, NULL); + g_debug ("Searching for data in: %s", metainfo_dir); + mi_files = as_utils_find_files_matching (metainfo_dir, "*.xml", FALSE, NULL); if (mi_files == NULL) { g_debug ("Unable find metainfo files."); return; @@ -1126,14 +1122,14 @@ as_pool_load_metainfo_data (AsPool *pool, GHashTable *desktop_entry_cpts) mi_cid_desktop = g_strdup_printf ("%s.desktop", mi_cid); /* check with .desktop suffix too */ - if (as_pool_has_component_id (pool, mi_cid_desktop, NULL)) { + if (as_component_registry_has_cid (registry, mi_cid_desktop)) { g_debug ("Skipped: %s (already known)", fname); continue; } } /* quickly check if we know the component already */ - if (as_pool_has_component_id (pool, mi_cid, NULL)) { + if (as_component_registry_has_cid (registry, mi_cid)) { g_debug ("Skipped: %s (already known)", fname); continue; } @@ -1161,8 +1157,8 @@ as_pool_load_metainfo_data (AsPool *pool, GHashTable *desktop_entry_cpts) if (cpt == NULL) continue; - /* we only read metainfo files from system directories */ - as_component_set_scope (cpt, AS_COMPONENT_SCOPE_SYSTEM); + /* set scope of these metainfo files */ + as_component_set_scope (cpt, scope); launchable = as_component_get_launchable (cpt, AS_LAUNCHABLE_KIND_DESKTOP_ID); if ((launchable != NULL) && (as_launchable_get_entries (launchable)->len > 0)) { @@ -1188,7 +1184,7 @@ as_pool_load_metainfo_data (AsPool *pool, GHashTable *desktop_entry_cpts) } } - as_pool_add_component_internal (pool, cpt, FALSE, &error); + as_pool_add_component_internal (pool, registry, cpt, FALSE, &error); if (error != NULL) { g_debug ("Component '%s' ignored: %s", as_component_get_data_id (cpt), error->message); g_error_free (error); @@ -1198,85 +1194,256 @@ as_pool_load_metainfo_data (AsPool *pool, GHashTable *desktop_entry_cpts) } /** - * as_pool_load_metainfo_desktop_data: + * as_pool_process_metainfo_desktop_data: * * Load metadata from metainfo files and .desktop files that - * where made available by locally installed applications. + * were made available by locally installed applications. */ static void -as_pool_load_metainfo_desktop_data (AsPool *pool) +as_pool_process_metainfo_desktop_data (AsPool *pool, + AsComponentRegistry *registry, + AsLocationGroup *lgroup, + const gchar *cache_key) { AsPoolPrivate *priv = GET_PRIVATE (pool); g_autoptr(GHashTable) de_cpts = NULL; g_autoptr(AsProfileTask) ptask = NULL; - ptask = as_profile_start_literal (priv->profile, "AsPool:load_metainfo_desktop_data"); - /* check if we actually need to load anything */ - g_mutex_lock (&priv->mutex); - if (!as_flags_contains (priv->flags, AS_POOL_FLAG_READ_DESKTOP_FILES) && !as_flags_contains (priv->flags, AS_POOL_FLAG_READ_METAINFO)) { - g_mutex_unlock (&priv->mutex); + if (!as_flags_contains (priv->flags, AS_POOL_FLAG_LOAD_OS_DESKTOP_FILES) && !as_flags_contains (priv->flags, AS_POOL_FLAG_LOAD_OS_METAINFO)) return; - } - g_mutex_unlock (&priv->mutex); - /* get a hashmap of desktop-entry components */ - de_cpts = as_pool_get_desktop_entries_table (pool); + /* check if the group has the right format */ + if (lgroup->format_style != AS_FORMAT_STYLE_METAINFO) + return; - g_mutex_lock (&priv->mutex); - if (as_flags_contains (priv->flags, AS_POOL_FLAG_READ_METAINFO)) { - g_mutex_unlock (&priv->mutex); + ptask = as_profile_start (priv->profile, "AsPool:load_metainfo_desktop_data:%s", cache_key); - /* load metainfo components, absorb desktop-entry components into them */ - as_pool_load_metainfo_data (pool, de_cpts); - } else { - g_mutex_unlock (&priv->mutex); + /* create a hashmap of all desktop-entry components we know of */ + de_cpts = g_hash_table_new_full (g_str_hash, + g_str_equal, + g_free, + (GDestroyNotify) g_object_unref); + for (guint i = 0; i < lgroup->locations->len; i++) { + AsLocationEntry *lentry = (AsLocationEntry*) g_ptr_array_index (lgroup->locations, i); + if (lentry->format_kind != AS_FORMAT_KIND_DESKTOP_ENTRY) + continue; + as_pool_update_desktop_entries_table (pool, de_cpts, lentry->location); + } + + if (as_flags_contains (priv->flags, AS_POOL_FLAG_LOAD_OS_METAINFO)) { + for (guint i = 0; i < lgroup->locations->len; i++) { + AsLocationEntry *lentry = (AsLocationEntry*) g_ptr_array_index (lgroup->locations, i); + if (lentry->format_kind != AS_FORMAT_KIND_XML) + continue; + + /* load metainfo components, absorb desktop-entry components into them */ + as_pool_load_metainfo_data (pool, + registry, + de_cpts, + lgroup->scope, + lentry->location, + cache_key); + } } /* read all remaining .desktop file components, if needed */ - g_mutex_lock (&priv->mutex); - if (as_flags_contains (priv->flags, AS_POOL_FLAG_READ_DESKTOP_FILES)) { + if (as_flags_contains (priv->flags, AS_POOL_FLAG_LOAD_OS_DESKTOP_FILES)) { GHashTableIter iter; gpointer value; GError *error = NULL; - g_mutex_unlock (&priv->mutex); - g_debug ("Including components from .desktop files in the pool."); - g_hash_table_iter_init (&iter, de_cpts); - while (g_hash_table_iter_next (&iter, NULL, &value)) { - AsComponent *cpt = AS_COMPONENT (value); + g_debug ("Adding components from desktop-entry files to the metadata pool."); + g_hash_table_iter_init (&iter, de_cpts); + while (g_hash_table_iter_next (&iter, NULL, &value)) { + AsComponent *cpt = AS_COMPONENT (value); + + as_pool_add_component_internal (pool, registry, cpt, FALSE, &error); + if (error != NULL) { + g_debug ("Ignored component '%s': %s", + as_component_get_data_id (cpt), error->message); + g_error_free (error); + error = NULL; + } + } + } +} + +/** + * as_pool_get_ctime: + * + * Returns: The changed time of a location. + */ +static time_t +as_get_location_ctime (const gchar *location) +{ + struct stat sb; + if (stat (location, &sb) < 0) + return 0; + return sb.st_ctime; +} + +/** + * as_pool_load_process_group: + * + * Process a location group and load all data from the source files + * or a suitable cache section. + */ +static gboolean +as_pool_load_process_group (AsPool *pool, + AsLocationGroup *lgroup, + const gchar *cache_key, + gboolean force_cache_refresh, + gboolean *caches_updated, + GError **error) +{ + AsPoolPrivate *priv = GET_PRIVATE (pool); + g_autoptr(AsComponentRegistry) registry = NULL; + g_autoptr(GPtrArray) final_results = NULL; + time_t cache_time; + gboolean cache_outdated = FALSE; + gboolean ret; + + if (lgroup->locations->len == 0) + return TRUE; - as_pool_add_component_internal (pool, cpt, FALSE, &error); - if (error != NULL) { - g_debug ("Component '%s' ignored: %s", as_component_get_data_id (cpt), error->message); - g_error_free (error); - error = NULL; + /* first check if we can load cache data */ + if (!force_cache_refresh && !as_flags_contains (priv->flags, AS_POOL_FLAG_IGNORE_CACHE_AGE)) { + cache_time = as_cache_get_ctime (priv->cache, + lgroup->scope, + cache_key, + NULL); + for (guint i = 0; i < lgroup->locations->len; i++) { + AsLocationEntry *lentry = (AsLocationEntry*) g_ptr_array_index (lgroup->locations, i); + if (as_get_location_ctime (lentry->location) > cache_time) { + cache_outdated = TRUE; + break; } } - } else { - g_mutex_unlock (&priv->mutex); + + if (!cache_outdated) { + /* cache is not out of data, let's use it! */ + g_debug ("Using cached metadata: %s", cache_key); + as_cache_load_section_for_key (priv->cache, + lgroup->scope, + lgroup->format_style, + lgroup->is_os_data, + cache_key, + &cache_outdated, + lgroup); + if (!cache_outdated) { + /* cache was fine and is now loaded, we are done here */ + return TRUE; + } + /* if we are here, the cache either went out of date (e.g. by being removed) + * or loading failed, in which case we will just regenerate it */ + g_debug ("Failed to load cache metadata for '%s' or cache suddenly went out of data. Regenerating cache.", + cache_key); + } } + + /* container for the generated components */ + registry = as_component_registry_new (); + + /* process any MetaInfo and desktop-entry files */ + as_pool_process_metainfo_desktop_data (pool, registry, + lgroup, + cache_key); + + /* process collection data - we intentionally ignore errors here, and just skip any broken metadata*/ + as_pool_load_collection_data (pool, + registry, + lgroup, + NULL); + + /* save cache section */ + final_results = as_component_registry_get_contents (registry); + ret = as_cache_set_contents_for_section (priv->cache, + lgroup->scope, + lgroup->format_style, + lgroup->is_os_data, + final_results, + cache_key, + lgroup, + error); + if (!ret) + return FALSE; + + /* we updated caches */ + if (caches_updated != NULL) + *caches_updated = TRUE; + + return TRUE; } /** - * as_pool_cache_refine_component_cb: + * as_pool_load_internal: * - * Callback function run on components before they are - * (de)serialized. + * Load metadata, either by reading the source files or loading + * an up-to-date cache of them. */ -static void -as_pool_cache_refine_component_cb (gpointer data, gpointer user_data) +static gboolean +as_pool_load_internal (AsPool *pool, + gboolean system_data_only, + gboolean force_cache_refresh, + gboolean *caches_updated, + GCancellable *cancellable, + GError **error) { - AsPool *pool = AS_POOL (user_data); AsPoolPrivate *priv = GET_PRIVATE (pool); - AsComponent *cpt = AS_COMPONENT (data); - g_autoptr(GMutexLocker) locker = g_mutex_locker_new (&priv->mutex); + g_autoptr(AsProfileTask) ptask = NULL; + GHashTableIter loc_iter; + gpointer loc_key, loc_value; + gboolean ret = TRUE; + + ptask = as_profile_start_literal (priv->profile, "AsPool:load"); + + /* load as AsPool also means to reload, so we clear any potential old data */ + if (!as_pool_clear2 (pool, error)) + return FALSE; + + /* apply settings */ + as_cache_set_prefer_os_metainfo (priv->cache, priv->prefer_local_metainfo); + + /* prune any ancient data from the cache that has not been used for a long time */ + as_cache_prune_data (priv->cache); + + /* find common locations that have metadata */ + as_pool_detect_std_metadata_dirs (pool, system_data_only); + + if (caches_updated != NULL) + *caches_updated = FALSE; + + /* process data from all the individual metadata silos in known locations */ + g_hash_table_iter_init (&loc_iter, priv->std_data_locations); + while (g_hash_table_iter_next (&loc_iter, &loc_key, &loc_value)) { + ret = as_pool_load_process_group (pool, + loc_value, + loc_key, + force_cache_refresh, + caches_updated, + error); + /* cache writing errors or other fatal stuff will cause us to stop loading anything */ + if (!ret) + return FALSE; + + } + + /* process data in user-defined locations */ + g_hash_table_iter_init (&loc_iter, priv->extra_data_locations); + while (g_hash_table_iter_next (&loc_iter, &loc_key, &loc_value)) { + ret = as_pool_load_process_group (pool, + loc_value, + loc_key, + force_cache_refresh, + caches_updated, + error); + if (!ret) + return FALSE; + + } - /* add additional data to the component, e.g. external screenshots. Also refines - * the component's icon paths */ - as_component_complete (cpt, - priv->screenshot_service_url, - priv->icon_dirs); + return ret; } /** @@ -1297,83 +1464,12 @@ as_pool_cache_refine_component_cb (gpointer data, gpointer user_data) gboolean as_pool_load (AsPool *pool, GCancellable *cancellable, GError **error) { - AsPoolPrivate *priv = GET_PRIVATE (pool); - g_autoptr(AsProfileTask) ptask = NULL; - gboolean ret = TRUE; - guint invalid_cpts_n; - gssize all_cpts_n; - gdouble valid_percentage; - GError *tmp_error = NULL; - - ptask = as_profile_start_literal (priv->profile, "AsPool:load"); - - g_mutex_lock (&priv->mutex); - if (as_flags_contains (priv->cache_flags, AS_CACHE_FLAG_NO_CLEAR)) { - g_autoptr(GMutexLocker) inner_locker = NULL; - g_mutex_unlock (&priv->mutex); - inner_locker = g_mutex_locker_new (&priv->mutex); - - /* we are supposed not to clear the cache before laoding its data */ - if (!as_cache_open (priv->cache, priv->cache_fname, priv->locale, error)) - return FALSE; - } else { - g_mutex_unlock (&priv->mutex); - - /* load (here) means to reload, so we clear potential old data */ - if (!as_pool_clear2 (pool, error)) - return FALSE; - } - - as_cache_make_floating (priv->cache); - - /* read all AppStream metadata that we can find */ - g_mutex_lock (&priv->mutex); - if (as_flags_contains (priv->flags, AS_POOL_FLAG_READ_COLLECTION)) { - g_mutex_unlock (&priv->mutex); - ret = as_pool_load_collection_data (pool, FALSE, error); - } else { - g_mutex_unlock (&priv->mutex); - } - - /* read all metainfo and desktop files and add them to the pool */ - as_pool_load_metainfo_desktop_data (pool); - - /* automatically refine the metadata we have in the pool */ - invalid_cpts_n = as_cache_unfloat (priv->cache, &tmp_error); - if (tmp_error != NULL) { - g_propagate_error (error, tmp_error); - return FALSE; - } - all_cpts_n = as_cache_count_components (priv->cache, &tmp_error); - if (all_cpts_n < 0) { - if (tmp_error != NULL) { - g_warning ("Unable to retrieve component count from cache: %s", tmp_error->message); - g_error_free (tmp_error); - tmp_error = NULL; - } - all_cpts_n = 0; - } - - valid_percentage = (all_cpts_n == 0)? 100 : (100 / (gdouble) all_cpts_n) * (gdouble) (all_cpts_n - invalid_cpts_n); - g_debug ("Percentage of valid components: %0.3f", valid_percentage); - - /* we only return a non-TRUE value if a significant amount (10%) of components has been declared invalid. */ - if ((invalid_cpts_n != 0) && (valid_percentage <= 90)) - ret = FALSE; - - /* report errors if refining has failed */ - if (!ret && (error != NULL)) { - if (*error == NULL) { - g_set_error_literal (error, - AS_POOL_ERROR, - AS_POOL_ERROR_INCOMPLETE, - _("Many components have been recognized as invalid. See debug output for details.")); - } else { - g_prefix_error (error, "Some components have been ignored: "); - } - } - - return ret; + return as_pool_load_internal (pool, + FALSE, /* all data, no just system data */ + FALSE, /* do not force cache refresh */ + NULL, /* we don't care whether caches were used or not */ + cancellable, + error); } static void @@ -1448,14 +1544,11 @@ as_pool_load_finish (AsPool *pool, gboolean as_pool_load_cache_file (AsPool *pool, const gchar *fname, GError **error) { - AsPoolPrivate *priv = GET_PRIVATE (pool); - g_autoptr(GMutexLocker) locker = g_mutex_locker_new (&priv->mutex); - - as_cache_close (priv->system_cache); - if (!as_cache_open (priv->cache, fname, priv->locale, error)) - return FALSE; - - return TRUE; + g_set_error (error, + AS_POOL_ERROR, + AS_POOL_ERROR_FAILED, + "Can not load cache file '%s': Direct cache injection is no longer possible.", fname); + return FALSE; } /** @@ -1469,24 +1562,11 @@ as_pool_load_cache_file (AsPool *pool, const gchar *fname, GError **error) gboolean as_pool_save_cache_file (AsPool *pool, const gchar *fname, GError **error) { - AsPoolPrivate *priv = GET_PRIVATE (pool); - g_autoptr(GPtrArray) cpts = NULL; - g_autoptr(AsCache) cache = NULL; - g_autoptr(GMutexLocker) locker = NULL; - - cpts = as_pool_get_components (pool); - cache = as_cache_new (); - locker = g_mutex_locker_new (&priv->mutex); - if (!as_cache_open (cache, fname, priv->locale, error)) - return FALSE; - g_clear_pointer (&locker, g_mutex_locker_free); - - for (guint i = 0; i < cpts->len; i++) { - if (!as_cache_insert (cache, AS_COMPONENT (g_ptr_array_index (cpts, i)), error)) - return FALSE; - } - - return TRUE; + g_set_error (error, + AS_POOL_ERROR, + AS_POOL_ERROR_FAILED, + "Can not write cache file '%s': Single-file cache export is no longer possible.", fname); + return FALSE; } /** @@ -1513,16 +1593,6 @@ as_pool_get_components (AsPool *pool) return g_ptr_array_new_with_free_func (g_object_unref); } - if (as_pool_can_query_system_cache (pool)) { - g_autoptr(GPtrArray) tmp_res = NULL; - tmp_res = as_cache_get_components_all (priv->system_cache, &tmp_error); - if (tmp_res == NULL) { - g_warning ("Unable to retrieve all components from system cache: %s", tmp_error->message); - return result; - } - as_object_ptr_array_absorb (result, tmp_res); - } - return result; } @@ -1542,28 +1612,9 @@ as_pool_get_components_by_id (AsPool *pool, const gchar *cid) { AsPoolPrivate *priv = GET_PRIVATE (pool); g_autoptr(AsProfileTask) ptask = NULL; - g_autoptr(GError) tmp_error = NULL; - GPtrArray *result = NULL; ptask = as_profile_start_literal (priv->profile, "AsPool:get_components_by_id"); - - result = as_cache_get_components_by_id (priv->cache, cid, &tmp_error); - if (result == NULL) { - g_warning ("Unable find components by ID in session cache: %s", tmp_error->message); - return g_ptr_array_new_with_free_func (g_object_unref); - } - - if (as_pool_can_query_system_cache (pool)) { - g_autoptr(GPtrArray) tmp_res = NULL; - tmp_res = as_cache_get_components_by_id (priv->system_cache, cid, &tmp_error); - if (tmp_res == NULL) { - g_warning ("Unable find components by ID in system cache: %s", tmp_error->message); - return result; - } - as_object_ptr_array_absorb (result, tmp_res); - } - - return result; + return as_cache_get_components_by_id (priv->cache, cid, NULL); } /** @@ -1591,16 +1642,6 @@ as_pool_get_components_by_provided_item (AsPool *pool, return g_ptr_array_new_with_free_func (g_object_unref); } - if (as_pool_can_query_system_cache (pool)) { - g_autoptr(GPtrArray) tmp_res = NULL; - tmp_res = as_cache_get_components_by_provided_item (priv->system_cache, kind, item, &tmp_error); - if (tmp_res == NULL) { - g_warning ("Unable find components by provided item in system cache: %s", tmp_error->message); - return result; - } - as_object_ptr_array_absorb (result, tmp_res); - } - return result; } @@ -1626,16 +1667,6 @@ as_pool_get_components_by_kind (AsPool *pool, AsComponentKind kind) return g_ptr_array_new_with_free_func (g_object_unref); } - if (as_pool_can_query_system_cache (pool)) { - g_autoptr(GPtrArray) tmp_res = NULL; - tmp_res = as_cache_get_components_by_kind (priv->system_cache, kind, &tmp_error); - if (tmp_res == NULL) { - g_warning ("Unable find components by kind in system cache: %s", tmp_error->message); - return result; - } - as_object_ptr_array_absorb (result, tmp_res); - } - return result; } @@ -1668,16 +1699,6 @@ as_pool_get_components_by_categories (AsPool *pool, gchar **categories) return g_ptr_array_new_with_free_func (g_object_unref); } - if (as_pool_can_query_system_cache (pool)) { - g_autoptr(GPtrArray) tmp_res = NULL; - tmp_res = as_cache_get_components_by_categories (priv->system_cache, categories, &tmp_error); - if (tmp_res == NULL) { - g_warning ("Unable find components by categories in system cache: %s", tmp_error->message); - return result; - } - as_object_ptr_array_absorb (result, tmp_res); - } - return result; } @@ -1709,16 +1730,6 @@ as_pool_get_components_by_launchable (AsPool *pool, return g_ptr_array_new_with_free_func (g_object_unref); } - if (as_pool_can_query_system_cache (pool)) { - g_autoptr(GPtrArray) tmp_res = NULL; - tmp_res = as_cache_get_components_by_launchable (priv->system_cache, kind, id, &tmp_error); - if (tmp_res == NULL) { - g_warning ("Unable find components by launchable in system cache: %s", tmp_error->message); - return result; - } - as_object_ptr_array_absorb (result, tmp_res); - } - return result; } @@ -1870,28 +1881,17 @@ as_pool_search (AsPool *pool, const gchar *search) g_debug ("Searching for: %s", tmp_str); } - result = as_cache_search (priv->cache, tokens, FALSE, &tmp_error); + result = as_cache_search (priv->cache, + tokens, /* sort */ + TRUE, + &tmp_error); if (result == NULL) { g_mutex_lock (&priv->mutex); - g_warning ("Search in session cache failed: %s", tmp_error->message); + g_warning ("Search failed: %s", tmp_error->message); g_mutex_unlock (&priv->mutex); return g_ptr_array_new_with_free_func (g_object_unref); } - if (as_pool_can_query_system_cache (pool)) { - g_autoptr(GPtrArray) tmp_res = NULL; - tmp_res = as_cache_search (priv->system_cache, tokens, FALSE, &tmp_error); - if (tmp_res == NULL) { - g_warning ("Search in system cache failed: %s", tmp_error->message); - return result; - } - as_object_ptr_array_absorb (result, tmp_res); - } - - /* sort the results by their priority (this was explicitly disabled for the caches before, - * so we could sort the combines result list) */ - as_sort_components_by_score (result); - return result; } @@ -1903,57 +1903,12 @@ as_pool_search (AsPool *pool, const gchar *search) * Update the AppStream cache. There is normally no need to call this function manually, because cache updates are handled * transparently in the background. * - * Returns: %TRUE if the cache was updated, %FALSE on error or if the cache update was not necessary and has been skipped. + * Returns: %TRUE on success, %FALSE on error. */ gboolean as_pool_refresh_cache (AsPool *pool, gboolean force, GError **error) { - return as_pool_refresh_system_cache (pool, FALSE, force, error); -} - -/** - * as_pool_cleanup_cache_dir: - * - * Delete all files in a cache directory. - */ -static void -as_pool_cleanup_cache_dir (AsPool *pool, const gchar *cache_dir) -{ - g_autoptr(GFileEnumerator) direnum = NULL; - g_autoptr(GFile) cdir = NULL; - g_autoptr(GError) error = NULL; - - cdir = g_file_new_for_path (cache_dir); - direnum = g_file_enumerate_children (cdir, - G_FILE_ATTRIBUTE_STANDARD_NAME, - G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, - NULL, - &error); - if (error != NULL) { - g_debug ("Unable to clean cache directories '%s': %s", cache_dir, error->message); - return; - } - - while (TRUE) { - GFileInfo *finfo = NULL; - g_autofree gchar *fname_full = NULL; - const gchar *fname; - if (!g_file_enumerator_iterate (direnum, &finfo, NULL, NULL, NULL)) - return; - if (finfo == NULL) - break; - if (g_file_info_get_file_type (finfo) != G_FILE_TYPE_REGULAR) - continue; - fname = g_file_info_get_name (finfo); - if (!g_str_has_suffix (fname, ".cache") && - !g_str_has_suffix (fname, ".tmp") && - !g_str_has_suffix (fname, ".mdb")) - continue; - - fname_full = g_build_filename (cache_dir, fname, NULL); - g_debug ("Deleting cache file: %s", fname); - g_unlink (fname_full); - } + return as_pool_refresh_system_cache (pool, FALSE, force, NULL, error); } /** @@ -1961,49 +1916,23 @@ as_pool_cleanup_cache_dir (AsPool *pool, const gchar *cache_dir) * @pool: An instance of #AsPool. * @user: Build cache for the current user, instead of system-wide. * @force: Enforce refresh, even if source data has not changed. + * @caches_updated: Return whether caches were updated or not. * * Update the AppStream cache. There is normally no need to call this function manually, because cache updates are handled * transparently in the background. * - * Returns: %TRUE if the cache was updated, %FALSE on error or if the cache update was not necessary and has been skipped. + * Returns: %TRUE on success, %FALSE on error. */ gboolean -as_pool_refresh_system_cache (AsPool *pool, gboolean user, gboolean force, GError **error) +as_pool_refresh_system_cache (AsPool *pool, + gboolean user, + gboolean force, + gboolean *caches_updated, + GError **error) { AsPoolPrivate *priv = GET_PRIVATE (pool); gboolean ret = FALSE; - gboolean cache_updated = FALSE; - g_autofree gchar *cache_fname = NULL; - g_autofree gchar *cache_fname_tmp = NULL; - g_autofree gchar *random_str = NULL; - g_autoptr(GError) data_load_error = NULL; GError *tmp_error = NULL; - AsCacheFlags prev_cache_flags; - guint invalid_cpts_n; - const gchar *sys_cache_dir; - - if (user) - sys_cache_dir = priv->sys_cache_dir_user; - else - sys_cache_dir = priv->sys_cache_dir_system; - - /* try to create cache directory, in case it doesn't exist */ - g_mkdir_with_parents (sys_cache_dir, 0755); - if (!as_utils_is_writable (sys_cache_dir)) { - g_set_error (error, - AS_POOL_ERROR, - AS_POOL_ERROR_TARGET_NOT_WRITABLE, - _("Cache location '%s' is not writable."), sys_cache_dir); - return FALSE; - } - - /* create the filename of our cache and set the location of the system cache. - * This has to happen before we check for new metadata, so the system cache can - * determine its age (so we know whether a refresh is needed at all). */ - g_mutex_lock (&priv->mutex); - cache_fname = g_strdup_printf ("%s/%s.cache", sys_cache_dir, priv->locale); - as_cache_set_location (priv->system_cache, cache_fname); - g_mutex_unlock (&priv->mutex); /* collect metadata */ #ifdef HAVE_APT_SUPPORT @@ -2018,113 +1947,23 @@ as_pool_refresh_system_cache (AsPool *pool, gboolean user, gboolean force, GErro } #endif - /* check if we need to refresh the cache - * (which is only necessary if the AppStream data has changed) */ - if (!as_pool_metadata_changed (pool, priv->system_cache, TRUE)) { - g_debug ("Data did not change, no cache refresh needed."); - if (force) { - g_debug ("Forcing refresh anyway."); - } else { - return FALSE; - } - } - g_debug ("Refreshing AppStream system data cache"); - - /* ensure we start with an empty pool */ - as_cache_close (priv->system_cache); - as_cache_close (priv->cache); - - /* don't call sync explicitly for a dramatic improvement in speed */ - as_cache_set_nosync (priv->cache, TRUE); - - /* open new system cache as user cache temporarily, so we can modify it */ - g_mutex_lock (&priv->mutex); - - random_str = as_random_alnum_string (8); - cache_fname_tmp = g_strconcat (cache_fname, random_str, ".tmp", NULL); - - /* remove old files for other languages in per-user mode */ - if (user) - as_pool_cleanup_cache_dir (pool, sys_cache_dir); - - if (!as_cache_open (priv->cache, cache_fname_tmp, priv->locale, error)) { - g_mutex_unlock (&priv->mutex); - g_remove (cache_fname_tmp); - return FALSE; - } - g_mutex_unlock (&priv->mutex); - - /* NOTE: we will only cache AppStream metadata, no .desktop file metadata etc. */ - - /* since the session cache is the system cache now (in order to update it), temporarily modify - * the cache flags */ - g_mutex_lock (&priv->mutex); - prev_cache_flags = priv->cache_flags; - priv->cache_flags = AS_CACHE_FLAG_USE_USER; - g_mutex_unlock (&priv->mutex); - - /* set cache to floating mode to increase performance by holding all data - * in memory in unserialized form */ - as_cache_make_floating (priv->cache); - - /* load AppStream collection metadata only and refine it */ - ret = as_pool_load_collection_data (pool, TRUE, &data_load_error); - if (data_load_error != NULL) - g_debug ("Error while updating the in-memory data pool: %s", data_load_error->message); - - /* un-float the cache, persisting all data */ - invalid_cpts_n = as_cache_unfloat (priv->cache, &tmp_error); - if (tmp_error != NULL) { - g_propagate_error (error, tmp_error); - g_remove (cache_fname_tmp); + /* ensure the cache is empty before refreshing data */ + as_cache_clear (priv->cache); + + /* load AppStream system metadata only and refine it */ + ret = as_pool_load_internal (pool, + TRUE, /* just system data */ + force, + caches_updated, + NULL, + &tmp_error); + if (!ret) { + g_propagate_prefixed_error (error, + tmp_error, + "Failed to refresh the metadata cache:"); return FALSE; } - /* save the cache object (this will sync it to disk explicitly too) */ - as_cache_close (priv->cache); - - /* atomically replace any old cache */ - g_chmod (cache_fname_tmp, 0644); - if (g_rename (cache_fname_tmp, cache_fname) < 0) - g_warning ("Unable to replace old cache '%s': %s", cache_fname, g_strerror (errno)); - else - cache_updated = TRUE; - g_clear_pointer (&cache_fname_tmp, g_free); - - /* restore cache flags */ - g_mutex_lock (&priv->mutex); - priv->cache_flags = prev_cache_flags; - g_mutex_unlock (&priv->mutex); - - /* switch back to default sync mode */ - as_cache_set_nosync (priv->cache, FALSE); - - /* reset (so the proper session cache is opened again) */ - as_pool_clear2 (pool, NULL); - - if (ret) { - if (invalid_cpts_n != 0) { - g_autofree gchar *error_message = NULL; - if (data_load_error == NULL) - error_message = g_strdup (_("The AppStream system cache was updated, but some components were ignored. Refer to the verbose log for more information.")); - else - error_message = g_strdup_printf (_("The AppStream system cache was updated, but problems were found which resulted in metadata being ignored: %s"), data_load_error->message); - - g_set_error_literal (error, - AS_POOL_ERROR, - AS_POOL_ERROR_INCOMPLETE, - error_message); - } - /* update the cache mtime, to not needlessly rebuild it again */ - if (cache_updated) - as_touch_location (cache_fname); - } else { - g_set_error (error, - AS_POOL_ERROR, - AS_POOL_ERROR_FAILED, - _("AppStream system cache refresh failed. Turn on verbose mode to get detailed issue information.")); - } - return TRUE; } @@ -2143,6 +1982,7 @@ as_pool_set_locale (AsPool *pool, const gchar *locale) g_free (priv->locale); priv->locale = g_strdup (locale); + as_cache_set_locale (priv->cache, locale); } /** @@ -2162,71 +2002,52 @@ as_pool_get_locale (AsPool *pool) } /** - * as_pool_add_metadata_location_internal: + * as_pool_add_extra_data_location: * @pool: An instance of #AsPool. * @directory: An existing filesystem location. - * @add_root: Whether to add the root directory if necessary. + * @format_style: The expected format style of the metadata, e.g. %AS_FORMAT_STYLE_COLLECTION * - * See %as_pool_add_metadata_location() + * Add an additional non-standard location to the metadata pool where metadata will be read from. + * If @directory contains a "xml", "xmls", "yaml" or "icons" subdirectory (or all of them), + * those paths will be added to the search paths instead. */ -static void -as_pool_add_metadata_location_internal (AsPool *pool, const gchar *directory, gboolean add_root) +void +as_pool_add_extra_data_location (AsPool *pool, const gchar *directory, AsFormatStyle format_style) { AsPoolPrivate *priv = GET_PRIVATE (pool); - gboolean dir_added = FALSE; - gchar *path; - g_autoptr(GMutexLocker) locker = NULL; - - if (!g_file_test (directory, G_FILE_TEST_IS_DIR)) { - g_debug ("Not adding metadata location '%s': Is no directory", directory); - return; - } - - /* protect access to directory arrays */ - locker = g_mutex_locker_new (&priv->mutex); - - /* metadata locations */ - path = g_build_filename (directory, "xml", NULL); - if (g_file_test (path, G_FILE_TEST_IS_DIR)) { - g_ptr_array_add (priv->xml_dirs, path); - dir_added = TRUE; - g_debug ("Added %s to XML metadata search path.", path); - } else { - g_free (path); - } - - path = g_build_filename (directory, "xmls", NULL); - if (g_file_test (path, G_FILE_TEST_IS_DIR)) { - g_ptr_array_add (priv->xml_dirs, path); - dir_added = TRUE; - g_debug ("Added %s to XML metadata search path.", path); - } else { - g_free (path); - } - - path = g_build_filename (directory, "yaml", NULL); - if (g_file_test (path, G_FILE_TEST_IS_DIR)) { - g_ptr_array_add (priv->yaml_dirs, path); - dir_added = TRUE; - g_debug ("Added %s to YAML metadata search path.", path); - } else { - g_free (path); - } + AsLocationGroup *extra_group; + extra_group = as_location_group_new (pool, + as_utils_guess_scope_from_path (directory), + format_style, + FALSE, /* is not OS data */ + directory); + g_hash_table_insert (priv->extra_data_locations, + g_strdup (extra_group->cache_key), + extra_group); + as_pool_add_collection_metadata_dir_internal (pool, + extra_group, + directory, + TRUE); +} - if ((add_root) && (!dir_added)) { - /* we didn't find metadata-specific directories, so let's watch to root path for both YAML and XML */ - g_ptr_array_add (priv->xml_dirs, g_strdup (directory)); - g_ptr_array_add (priv->yaml_dirs, g_strdup (directory)); - g_debug ("Added %s to all metadata search paths.", directory); - } +/** + * as_pool_reset_extra_data_locations: + * @pool: An instance of #AsPool. + * + * Remove all explicitly added metadata locations. + * + * Since: 0.14.7 + */ +void +as_pool_reset_extra_data_locations (AsPool *pool) +{ + AsPoolPrivate *priv = GET_PRIVATE (pool); + g_autoptr(GMutexLocker) locker = g_mutex_locker_new (&priv->mutex); - /* icons */ - path = g_build_filename (directory, "icons", NULL); - if (g_file_test (path, G_FILE_TEST_IS_DIR)) - g_ptr_array_add (priv->icon_dirs, path); - else - g_free (path); + /* clear arrays */ + g_hash_table_remove_all (priv->extra_data_locations); + g_debug ("Cleared extra metadata search paths."); } /** @@ -2237,11 +2058,13 @@ as_pool_add_metadata_location_internal (AsPool *pool, const gchar *directory, gb * Add a location for the data pool to read data from. * If @directory contains a "xml", "xmls", "yaml" or "icons" subdirectory (or all of them), * those paths will be added to the search paths instead. + * + * Deprecated: 0.14.7: Use %as_pool_add_extra_data_location instead. */ void as_pool_add_metadata_location (AsPool *pool, const gchar *directory) { - as_pool_add_metadata_location_internal (pool, directory, TRUE); + as_pool_add_extra_data_location (pool, directory, AS_FORMAT_STYLE_COLLECTION); } /** @@ -2249,6 +2072,8 @@ as_pool_add_metadata_location (AsPool *pool, const gchar *directory) * @pool: An instance of #AsPool. * * Remove all metadata locations from the list of watched locations. + * + * Deprecated: 0.14.7: Use %as_pool_reset_extra_data_locations and control system data loading via flags. */ void as_pool_clear_metadata_locations (AsPool *pool) @@ -2256,10 +2081,13 @@ as_pool_clear_metadata_locations (AsPool *pool) AsPoolPrivate *priv = GET_PRIVATE (pool); g_autoptr(GMutexLocker) locker = g_mutex_locker_new (&priv->mutex); + as_flags_remove (priv->flags, AS_POOL_FLAG_LOAD_OS_COLLECTION); + as_flags_remove (priv->flags, AS_POOL_FLAG_LOAD_OS_METAINFO); + as_flags_remove (priv->flags, AS_POOL_FLAG_LOAD_OS_DESKTOP_FILES); + as_flags_remove (priv->flags, AS_POOL_FLAG_LOAD_FLATPAK); + /* clear arrays */ - g_ptr_array_set_size (priv->xml_dirs, 0); - g_ptr_array_set_size (priv->yaml_dirs, 0); - g_ptr_array_set_size (priv->icon_dirs, 0); + g_hash_table_remove_all (priv->extra_data_locations); g_debug ("Cleared all metadata search paths."); } @@ -2269,13 +2097,15 @@ as_pool_clear_metadata_locations (AsPool *pool) * @pool: An instance of #AsPool. * * Get the #AsCacheFlags for this data pool. + * + * Deprecated: 0.14.7: Cache flags can no longer be changed. */ AsCacheFlags as_pool_get_cache_flags (AsPool *pool) { AsPoolPrivate *priv = GET_PRIVATE (pool); g_autoptr(GMutexLocker) locker = g_mutex_locker_new (&priv->mutex); - return priv->cache_flags; + return AS_CACHE_FLAG_USE_SYSTEM | AS_CACHE_FLAG_USE_USER | AS_CACHE_FLAG_REFRESH_SYSTEM; } /** @@ -2284,13 +2114,12 @@ as_pool_get_cache_flags (AsPool *pool) * @flags: The new #AsCacheFlags. * * Set the #AsCacheFlags for this data pool. + * + * Deprecated: 0.14.7: Cache flags can no longer be modified. */ void as_pool_set_cache_flags (AsPool *pool, AsCacheFlags flags) { - AsPoolPrivate *priv = GET_PRIVATE (pool); - g_autoptr(GMutexLocker) locker = g_mutex_locker_new (&priv->mutex); - priv->cache_flags = flags; } /** @@ -2323,16 +2152,19 @@ as_pool_set_flags (AsPool *pool, AsPoolFlags flags) } /** - * as_pool_get_system_cache_age: + * as_pool_get_os_metadata_cache_age: * @pool: An instance of #AsPool. * - * Get the age of the system cache. + * Get the age of the system cache for OS collection data. */ time_t -as_pool_get_system_cache_age (AsPool *pool) +as_pool_get_os_metadata_cache_age (AsPool *pool) { AsPoolPrivate *priv = GET_PRIVATE (pool); - return as_cache_get_ctime (priv->system_cache); + return as_cache_get_ctime (priv->cache, + AS_COMPONENT_SCOPE_SYSTEM, + OS_COLLECTION_CACHE_KEY, + NULL); } /** @@ -2345,15 +2177,13 @@ as_pool_get_system_cache_age (AsPool *pool) * a temporary directory. In any other case, the given filename is used. * * Since: 0.12.7 + * + * Deprecated: 0.14.7: Cache location can no longer be set explicitly. **/ void as_pool_set_cache_location (AsPool *pool, const gchar *fname) { - AsPoolPrivate *priv = GET_PRIVATE (pool); - g_autoptr(GMutexLocker) locker = g_mutex_locker_new (&priv->mutex); - - g_free (priv->cache_fname); - priv->cache_fname = g_strdup (fname); + g_warning ("Not changing AppStream cache location: No longer supported."); } /** @@ -2362,14 +2192,36 @@ as_pool_set_cache_location (AsPool *pool, const gchar *fname) * * Gets the location of the session cache. * - * Returns: Location of the cache. + * Returns: Location of the cache, or %NULL if unknown. + * + * Deprecated: 0.14.7: Cache location can no longer be set explicitly. **/ const gchar* as_pool_get_cache_location (AsPool *pool) +{ + /* No-op */ + return NULL; +} + +/** + * as_pool_override_cache_location: + * @pool: An instance of #AsPool. + * @dir_sys: Directory to store system/non-writable cache + * @dir_user: Directory to store writable/user cache. + * + * Override the automatic cache location placement. + * This is useful primarily for debugging purposes and unit tests. + **/ +void +as_pool_override_cache_locations (AsPool *pool, const gchar *dir_sys, const gchar *dir_user) { AsPoolPrivate *priv = GET_PRIVATE (pool); - g_autoptr(GMutexLocker) locker = g_mutex_locker_new (&priv->mutex); - return priv->cache_fname; + if (dir_sys == NULL) + as_cache_set_locations (priv->cache, dir_user, dir_user); + else if (dir_user == NULL) + as_cache_set_locations (priv->cache, dir_sys, dir_sys); + else + as_cache_set_locations (priv->cache, dir_sys, dir_user); } /** diff --git a/src/as-pool.h b/src/as-pool.h index 2929e4116..29e5705da 100644 --- a/src/as-pool.h +++ b/src/as-pool.h @@ -67,19 +67,27 @@ typedef enum { /** * AsPoolFlags: * @AS_POOL_FLAG_NONE: No flags. - * @AS_POOL_FLAG_READ_COLLECTION: Add AppStream collection metadata to the pool. - * @AS_POOL_FLAG_READ_METAINFO: Add data from AppStream metainfo files to the pool. - * @AS_POOL_FLAG_READ_DESKTOP_FILES: Add metadata from .desktop files to the pool. + * @AS_POOL_FLAG_LOAD_OS_COLLECTION: Load AppStream collection metadata from OS locations. + * @AS_POOL_FLAG_LOAD_OS_METAINFO: Load MetaInfo data from OS locations. + * @AS_POOL_FLAG_LOAD_OS_DESKTOP_FILES: Load components from desktop-entry files in OS locations. + * @AS_POOL_FLAG_LOAD_FLATPAK: Load AppStream collection metadata from Flatpak. + * @AS_POOL_FLAG_IGNORE_CACHE_AGE: Load fresh data even if an up-o-date cache is available. * * Flags on how caching should be used. **/ typedef enum { AS_POOL_FLAG_NONE = 0, - AS_POOL_FLAG_READ_COLLECTION = 1 << 0, - AS_POOL_FLAG_READ_METAINFO = 1 << 1, - AS_POOL_FLAG_READ_DESKTOP_FILES = 1 << 2, + AS_POOL_FLAG_LOAD_OS_COLLECTION = 1 << 0, + AS_POOL_FLAG_LOAD_OS_METAINFO = 1 << 1, + AS_POOL_FLAG_LOAD_OS_DESKTOP_FILES = 1 << 2, + AS_POOL_FLAG_LOAD_FLATPAK = 1 << 3, + AS_POOL_FLAG_IGNORE_CACHE_AGE = 1 << 4, } AsPoolFlags; +#define AS_POOL_FLAG_READ_COLLECTION AS_POOL_FLAG_LOAD_OS_COLLECTION +#define AS_POOL_FLAG_READ_METAINFO AS_POOL_FLAG_LOAD_OS_METAINFO +#define AS_POOL_FLAG_READ_DESKTOP_FILES AS_POOL_FLAG_LOAD_OS_DESKTOP_FILES + /** * AsPoolError: * @AS_POOL_ERROR_FAILED: Generic failure @@ -122,6 +130,9 @@ gboolean as_pool_load_finish (AsPool *pool, gboolean as_pool_clear2 (AsPool *pool, GError **error); +gboolean as_pool_add_components (AsPool *pool, + GPtrArray *cpts, + GError **error); gboolean as_pool_add_component (AsPool *pool, AsComponent *cpt, GError **error); @@ -144,22 +155,15 @@ GPtrArray *as_pool_search (AsPool *pool, gchar **as_pool_build_search_tokens (AsPool *pool, const gchar *search); -void as_pool_clear_metadata_locations (AsPool *pool); -void as_pool_add_metadata_location (AsPool *pool, - const gchar *directory); - -AsCacheFlags as_pool_get_cache_flags (AsPool *pool); -void as_pool_set_cache_flags (AsPool *pool, - AsCacheFlags flags); +void as_pool_reset_extra_data_locations (AsPool *pool); +void as_pool_add_extra_data_location (AsPool *pool, + const gchar *directory, + AsFormatStyle format_style); AsPoolFlags as_pool_get_flags (AsPool *pool); void as_pool_set_flags (AsPool *pool, AsPoolFlags flags); -const gchar *as_pool_get_cache_location (AsPool *pool); -void as_pool_set_cache_location (AsPool *pool, - const gchar *fname); - /* DEPRECATED */ G_DEPRECATED @@ -177,6 +181,24 @@ gboolean as_pool_refresh_cache (AsPool *pool, G_DEPRECATED void as_pool_clear (AsPool *pool); +G_DEPRECATED +const gchar *as_pool_get_cache_location (AsPool *pool); +G_DEPRECATED +void as_pool_set_cache_location (AsPool *pool, + const gchar *fname); + +G_DEPRECATED +AsCacheFlags as_pool_get_cache_flags (AsPool *pool); +G_DEPRECATED +void as_pool_set_cache_flags (AsPool *pool, + AsCacheFlags flags); + +G_DEPRECATED +void as_pool_clear_metadata_locations (AsPool *pool); +G_DEPRECATED +void as_pool_add_metadata_location (AsPool *pool, + const gchar *directory); + G_END_DECLS #endif /* __AS_POOL_H */ diff --git a/src/as-settings-private.h b/src/as-settings-private.h index cfe92aff7..3e7d4b485 100644 --- a/src/as-settings-private.h +++ b/src/as-settings-private.h @@ -29,7 +29,7 @@ G_BEGIN_DECLS #define AS_INTERNAL_VISIBLE __attribute__((visibility("default"))) #define AS_CONFIG_NAME "/etc/appstream.conf" -#define AS_APPSTREAM_CACHE_PATH "/var/cache/app-info/cache" +#define AS_APPSTREAM_SYS_CACHE_DIR "/var/cache/app-info/cache" /* declared in as-data-pool.c */ AS_INTERNAL_VISIBLE diff --git a/src/as-tag-xml.gperf b/src/as-tag-xml.gperf index 3a3ac46fc..7e208c2df 100644 --- a/src/as-tag-xml.gperf +++ b/src/as-tag-xml.gperf @@ -48,3 +48,4 @@ ol, AS_TAG_OL __asi_scope, AS_TAG_INTERNAL_SCOPE __asi_origin, AS_TAG_INTERNAL_ORIGIN __asi_branch, AS_TAG_INTERNAL_BRANCH +__asi_tokens, AS_TAG_INTERNAL_TOKENS diff --git a/src/as-tag.h b/src/as-tag.h index a2e3a2b18..280de778e 100644 --- a/src/as-tag.h +++ b/src/as-tag.h @@ -120,6 +120,7 @@ typedef enum { AS_TAG_INTERNAL_SCOPE, AS_TAG_INTERNAL_ORIGIN, AS_TAG_INTERNAL_BRANCH, + AS_TAG_INTERNAL_TOKENS, /*< private >*/ AS_TAG_LAST diff --git a/src/as-utils-private.h b/src/as-utils-private.h index 9e15e0e9f..4df3fb16e 100644 --- a/src/as-utils-private.h +++ b/src/as-utils-private.h @@ -114,6 +114,7 @@ GPtrArray *as_utils_find_files (const gchar *dir, gboolean recursive, GError **error); +AS_INTERNAL_VISIBLE gboolean as_utils_is_root (void); AS_INTERNAL_VISIBLE @@ -132,6 +133,8 @@ void as_hash_table_string_keys_to_array (GHashTable *table, GPtrArray *array); gboolean as_touch_location (const gchar *fname); + +AS_INTERNAL_VISIBLE void as_reset_umask (void); AS_INTERNAL_VISIBLE diff --git a/src/as-utils.c b/src/as-utils.c index 1f49d75a9..56936fd21 100644 --- a/src/as-utils.c +++ b/src/as-utils.c @@ -2436,3 +2436,21 @@ as_utils_find_stock_icon_filename_full (const gchar *root_dir, "Failed to find icon %s", icon_name); return NULL; } + +/** + * as_utils_guess_scope_from_path: + * @path: The filename to test. + * + * Guess the #AsComponentScope that applies to a given path. + * + * Returns: the #AsComponentScope + * + * Since: 0.14.8 + */ +AsComponentScope +as_utils_guess_scope_from_path (const gchar *path) +{ + if (g_str_has_prefix (path, "/home") || g_str_has_prefix (path, g_get_home_dir ())) + return AS_COMPONENT_SCOPE_USER; + return AS_COMPONENT_SCOPE_SYSTEM; +} diff --git a/src/as-utils.h b/src/as-utils.h index 35235964b..515f2e414 100644 --- a/src/as-utils.h +++ b/src/as-utils.h @@ -132,6 +132,8 @@ gboolean as_utils_install_metadata_file (AsMetadataLocation location, const gchar *destdir, GError **error); +AsComponentScope as_utils_guess_scope_from_path (const gchar *path); + /* DEPRECATED */ diff --git a/src/as-xml.c b/src/as-xml.c index 277b96a4f..7cce91785 100644 --- a/src/as-xml.c +++ b/src/as-xml.c @@ -1117,12 +1117,14 @@ as_xml_node_to_str (xmlNode *root, GError **error) if (error_msg_str != NULL) { if (error == NULL) { g_warning ("Could not serialize XML document: %s", error_msg_str); + g_free (g_steal_pointer (&xmlstr)); goto out; } else { g_set_error (error, AS_METADATA_ERROR, AS_METADATA_ERROR_FAILED, "Could not serialize XML document: %s", error_msg_str); + g_free (g_steal_pointer (&xmlstr)); goto out; } } diff --git a/src/meson.build b/src/meson.build index 8a35a51cb..839f4a78e 100644 --- a/src/meson.build +++ b/src/meson.build @@ -162,12 +162,9 @@ aslib_deps = [glib_dep, gobject_dep, gio_unix_dep, curl_dep, - lmdb_dep, + xmlb_dep, xml2_dep, yaml_dep] -if not lmdb_dep.found() - aslib_deps += [lmdb_lib] -endif if get_option ('stemming') aslib_deps += [stemmer_lib] endif diff --git a/tests/test-performance.c b/tests/test-performance.c index 0612ed2df..398226f35 100644 --- a/tests/test-performance.c +++ b/tests/test-performance.c @@ -23,6 +23,7 @@ #include #include "appstream.h" +#include "as-utils-private.h" #include "as-metadata.h" #include "as-test-utils.h" #include "as-pool-private.h" @@ -47,17 +48,18 @@ test_get_sampledata_pool (gboolean use_caches) mdata_dir = g_build_filename (datadir, "collection", NULL); pool = as_pool_new (); - as_pool_clear_metadata_locations (pool); - as_pool_add_metadata_location (pool, mdata_dir); as_pool_set_locale (pool, "C"); flags = as_pool_get_flags (pool); - as_flags_remove (flags, AS_POOL_FLAG_READ_DESKTOP_FILES); - as_flags_remove (flags, AS_POOL_FLAG_READ_METAINFO); + as_flags_remove (flags, AS_POOL_FLAG_LOAD_OS_COLLECTION); + as_flags_remove (flags, AS_POOL_FLAG_LOAD_OS_DESKTOP_FILES); + as_flags_remove (flags, AS_POOL_FLAG_LOAD_OS_METAINFO); + as_flags_remove (flags, AS_POOL_FLAG_LOAD_FLATPAK); + if (!use_caches) + as_flags_add (flags, AS_POOL_FLAG_IGNORE_CACHE_AGE); as_pool_set_flags (pool, flags); - if (!use_caches) - as_pool_set_cache_flags (pool, AS_CACHE_FLAG_NONE); + as_pool_add_extra_data_location (pool, mdata_dir, AS_FORMAT_STYLE_COLLECTION); return pool; } @@ -100,15 +102,18 @@ test_pool_cache_perf (void) g_autoptr(GPtrArray) prep_cpts = NULL; g_autoptr(AsCache) cache = NULL; g_auto(GStrv) strv = NULL; + g_autofree gchar *mdata_dir = NULL; guint loops = 1000; - const gchar *cache_location = "/tmp/as-unittest-perfcache.mdb"; + const gchar *cache_location = "/tmp/as-unittest-perfcache"; + mdata_dir = g_build_filename (datadir, "collection", NULL); /* prepare a cache file and list of components to work with */ + as_utils_delete_dir_recursive (cache_location); { g_autoptr(AsPool) prep_pool = NULL; prep_pool = test_get_sampledata_pool (FALSE); - as_pool_set_cache_location (prep_pool, cache_location); + as_pool_override_cache_locations (prep_pool, cache_location, NULL); as_pool_load (prep_pool, NULL, &error); g_assert_no_error (error); @@ -116,63 +121,53 @@ test_pool_cache_perf (void) g_assert_cmpint (prep_cpts->len, ==, 19); } + /* test fetching all components from cache */ timer = g_timer_new (); for (guint i = 0; i < loops; i++) { g_autoptr(GPtrArray) cpts = NULL; - AsPoolFlags flags; - g_autoptr(AsPool) pool = as_pool_new (); - - as_pool_clear_metadata_locations (pool); - as_pool_set_locale (pool, "C"); - as_pool_set_cache_location (pool, cache_location); + g_autoptr(AsCache) tmp_cache = as_cache_new (); - flags = as_pool_get_flags (pool); - as_flags_remove (flags, AS_POOL_FLAG_READ_DESKTOP_FILES); - as_flags_remove (flags, AS_POOL_FLAG_READ_METAINFO); - as_pool_set_flags (pool, flags); - as_pool_set_cache_flags (pool, as_pool_get_cache_flags (pool) | AS_CACHE_FLAG_NO_CLEAR); + as_cache_set_locale (tmp_cache, "C"); + as_cache_set_locations (tmp_cache, cache_location, cache_location); + as_cache_load_section_for_path (tmp_cache, mdata_dir, NULL, NULL); - as_pool_load (pool, NULL, &error); + cpts = as_cache_get_components_all (tmp_cache, &error); g_assert_no_error (error); - - cpts = as_pool_get_components (pool); g_assert_cmpint (cpts->len, ==, 19); } g_print ("\n Cache readall: %.2f ms", g_timer_elapsed (timer, NULL) * 1000 / loops); - g_assert_cmpint (g_remove (cache_location), ==, 0); + g_assert_true (as_utils_delete_dir_recursive (cache_location)); /* test cache write speed */ g_timer_reset (timer); for (guint i = 0; i < loops; i++) { g_autoptr(AsCache) tmp_cache = as_cache_new (); - as_cache_set_nosync (tmp_cache, TRUE); - as_cache_open (tmp_cache, cache_location, "C", &error); - g_assert_no_error (error); + as_cache_set_locale (tmp_cache, "C"); - for (guint i = 0; i < prep_cpts->len; i++) { - AsComponent *cpt = AS_COMPONENT (g_ptr_array_index (prep_cpts, i)); - as_cache_insert (tmp_cache, cpt, &error); - g_assert_no_error (error); - } + as_cache_set_contents_for_path (tmp_cache, + prep_cpts, + "dummy", + NULL, + &error); + g_assert_no_error (error); - g_assert_cmpint (g_remove (cache_location), ==, 0); + g_assert_true (as_utils_delete_dir_recursive (cache_location)); } g_print ("\n Cache write: %.2f ms", g_timer_elapsed (timer, NULL) * 1000 / loops); /* test search */ cache = as_cache_new (); - as_cache_open (cache, cache_location, "C", &error); + as_cache_set_locale (cache, "C"); + as_cache_set_contents_for_path (cache, + prep_cpts, + "dummy", + NULL, + &error); g_assert_no_error (error); - for (guint i = 0; i < prep_cpts->len; i++) { - AsComponent *cpt = AS_COMPONENT (g_ptr_array_index (prep_cpts, i)); - as_cache_insert (cache, cpt, &error); - g_assert_no_error (error); - } - - strv = g_strsplit ("gam\namateur", "\n", -1); + strv = g_strsplit ("gam|amateur", "|", -1); g_timer_reset (timer); for (guint i = 0; i < loops; i++) { g_autoptr(GPtrArray) test_cpts = NULL; diff --git a/tests/test-pool.c b/tests/test-pool.c index eb33747c8..7214f901b 100644 --- a/tests/test-pool.c +++ b/tests/test-pool.c @@ -74,22 +74,28 @@ test_get_sampledata_pool (gboolean use_caches) AsPool *pool; AsPoolFlags flags; g_autofree gchar *mdata_dir = NULL; + g_autofree gchar *cache_dummy_dir = NULL; /* create AsPool and load sample metadata */ mdata_dir = g_build_filename (datadir, "collection", NULL); pool = as_pool_new (); - as_pool_clear_metadata_locations (pool); - as_pool_add_metadata_location (pool, mdata_dir); + as_pool_add_extra_data_location (pool, mdata_dir, AS_FORMAT_STYLE_COLLECTION); as_pool_set_locale (pool, "C"); flags = as_pool_get_flags (pool); - as_flags_remove (flags, AS_POOL_FLAG_READ_DESKTOP_FILES); - as_flags_remove (flags, AS_POOL_FLAG_READ_METAINFO); - as_pool_set_flags (pool, flags); + as_flags_remove (flags, AS_POOL_FLAG_LOAD_OS_COLLECTION); + as_flags_remove (flags, AS_POOL_FLAG_LOAD_OS_DESKTOP_FILES); + as_flags_remove (flags, AS_POOL_FLAG_LOAD_OS_METAINFO); + as_flags_remove (flags, AS_POOL_FLAG_LOAD_FLATPAK); if (!use_caches) - as_pool_set_cache_flags (pool, AS_CACHE_FLAG_NONE); + as_flags_add (flags, AS_POOL_FLAG_IGNORE_CACHE_AGE); + as_pool_set_flags (pool, flags); + + cache_dummy_dir = g_build_filename (g_get_tmp_dir (), "as-cache-dummy", "XXXXXX", NULL); + g_mkdtemp (cache_dummy_dir); + as_pool_override_cache_locations (pool, cache_dummy_dir, NULL); return pool; } @@ -157,13 +163,14 @@ test_cache () g_autoptr(AsCache) cache = NULL; g_autoptr(AsComponent) ccpt = NULL; g_autoptr(GError) error = NULL; + g_autofree gchar *mdata_dir = NULL; g_autofree gchar *xmldata_precache = NULL; g_autofree gchar *xmldata_postcache = NULL; - gboolean ret; - static const gchar *cache_testpath = "/tmp/as-unittest-cache.cache"; + + static const gchar *cache_testpath = "/tmp/as-unittest-tmpcache"; pool = test_get_sampledata_pool (FALSE); - as_pool_set_cache_location (pool, cache_testpath); + as_pool_override_cache_locations (pool, cache_testpath, NULL); as_pool_load (pool, NULL, &error); g_assert_no_error (error); @@ -190,16 +197,15 @@ test_cache () g_assert_no_error (error); /* ensure we get the same result back that we cached before */ - g_object_unref (pool); - pool = test_get_sampledata_pool (FALSE); - as_pool_clear_metadata_locations (pool); - as_pool_set_cache_flags (pool, as_pool_get_cache_flags (pool) | AS_CACHE_FLAG_NO_CLEAR); + mdata_dir = g_build_filename (datadir, "collection", NULL); + cache = as_cache_new (); + as_cache_set_locale (cache, "C"); - as_pool_set_cache_location (pool, cache_testpath); - as_pool_load (pool, NULL, &error); - g_assert_no_error (error); + as_cache_set_locations (cache, cache_testpath, cache_testpath); + as_cache_load_section_for_path (cache, mdata_dir, NULL, NULL); - cpts_post = as_pool_get_components (pool); + cpts_post = as_cache_get_components_all (cache, &error); + g_assert_no_error (error); g_assert_cmpint (cpts_post->len, ==, 19); as_assert_component_lists_equal (cpts_post, cpts_prev); @@ -213,27 +219,6 @@ test_cache () xmldata_postcache = as_metadata_components_to_collection (mdata, AS_FORMAT_KIND_XML, &error); g_assert_no_error (error); g_assert_true (as_test_compare_lines (xmldata_precache, xmldata_postcache)); - - /* load an "modify" read-only cache */ - cache = as_cache_new (); - as_cache_set_readonly (cache, TRUE); - as_cache_open (cache, cache_testpath, "C", &error); - g_assert_no_error (error); - - ccpt = as_cache_get_component_by_data_id (cache, "system/package/os/org.inkscape.Inkscape/*", &error); - g_assert_no_error (error); - g_assert_nonnull (ccpt); - - g_assert_cmpstr (as_component_get_name (ccpt), ==, "Inkscape"); - g_object_unref (ccpt); - - ret = as_cache_remove_by_data_id (cache, "system/package/os/org.inkscape.Inkscape/*", &error); - g_assert_no_error (error); - g_assert_true (ret); - - ccpt = as_cache_get_component_by_data_id (cache, "system/package/os/org.inkscape.Inkscape/*", &error); - g_assert_no_error (error); - g_assert_null (ccpt); } /** @@ -604,7 +589,7 @@ test_pool_empty () gboolean ret; pool = as_pool_new (); - as_pool_clear_metadata_locations (pool); + as_pool_reset_extra_data_locations (pool); as_pool_set_locale (pool, "C"); /* test reading from the pool when it wasn't loaded yet */ diff --git a/tools/appstreamcli.c b/tools/appstreamcli.c index bd33c87af..4cb63697a 100644 --- a/tools/appstreamcli.c +++ b/tools/appstreamcli.c @@ -26,6 +26,7 @@ #include #include "as-profile.h" +#include "as-utils-private.h" #include "ascli-utils.h" #include "ascli-actions-mdata.h" @@ -1291,13 +1292,22 @@ as_client_run (char **argv, int argc) optn_nonet = TRUE; } + /* set some global defaults, in case we run as root in an unsafe environment */ + if (as_utils_is_root ()) { + /* users umask shouldn't interfere with us creating new files when we are root */ + as_reset_umask (); + + /* ensure we never start gvfsd as root: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=852696 */ + g_setenv ("GIO_USE_VFS", "local", TRUE); + } + ascli_set_output_colored (!optn_no_color); /* if out terminal is no tty, disable colors automatically */ if (!isatty (fileno (stdout))) ascli_set_output_colored (FALSE); - /* don't let gvfsd start it's own session bus: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=852696 */ + /* don't let gvfsd start its own session bus: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=852696 */ g_setenv ("GIO_USE_VFS", "local", TRUE); /* prepare profiler */ diff --git a/tools/ascli-actions-mdata.c b/tools/ascli-actions-mdata.c index b48499774..aa2ebb8f5 100644 --- a/tools/ascli-actions-mdata.c +++ b/tools/ascli-actions-mdata.c @@ -37,34 +37,35 @@ ascli_refresh_cache (const gchar *cachepath, const gchar *datapath, gboolean use { g_autoptr(AsPool) dpool = NULL; g_autoptr(GError) error = NULL; + gboolean cache_updated; gboolean ret = FALSE; dpool = as_pool_new (); if (datapath != NULL) { AsPoolFlags flags; - /* the user wants data from a different path to be used */ - as_pool_clear_metadata_locations (dpool); - as_pool_add_metadata_location (dpool, datapath); - /* we auto-disable loading data from sources that are not in datapath for now */ flags = as_pool_get_flags (dpool); - as_flags_remove (flags, AS_POOL_FLAG_READ_DESKTOP_FILES); - as_flags_remove (flags, AS_POOL_FLAG_READ_METAINFO); + as_flags_remove (flags, AS_POOL_FLAG_LOAD_OS_DESKTOP_FILES); + as_flags_remove (flags, AS_POOL_FLAG_LOAD_OS_METAINFO); + as_flags_remove (flags, AS_POOL_FLAG_LOAD_FLATPAK); as_pool_set_flags (dpool, flags); - /* disable loading data from cache */ - as_pool_set_cache_flags (dpool, AS_CACHE_FLAG_NONE); + /* the user wants data from a different path to be used */ + as_pool_add_extra_data_location (dpool, + datapath, + AS_FORMAT_STYLE_COLLECTION); } if (cachepath == NULL) { - ret = as_pool_refresh_system_cache (dpool, user, forced, &error); + ret = as_pool_refresh_system_cache (dpool, user, forced, &cache_updated, &error); } else { - as_pool_set_cache_location (dpool, cachepath); + as_pool_override_cache_locations (dpool, cachepath, cachepath); as_pool_load (dpool, NULL, &error); + cache_updated = TRUE; } - if (error != NULL) { + if (!ret) { if (g_error_matches (error, AS_POOL_ERROR, AS_POOL_ERROR_TARGET_NOT_WRITABLE)) /* TRANSLATORS: In ascli: The requested action needs higher permissions. */ g_printerr ("%s\n%s\n", error->message, _("You might need superuser permissions to perform this action.")); @@ -73,19 +74,15 @@ ascli_refresh_cache (const gchar *cachepath, const gchar *datapath, gboolean use return 2; } - if (ret) { - /* we performed a cache refresh */ - + if (cache_updated) { /* TRANSLATORS: Updating the metadata cache succeeded */ g_print ("%s\n", _("AppStream cache update completed successfully.")); - - /* no > 0 error code, since we updated something */ - return 0; } else { - /* cache wasn't updated, so the update wasn't necessary */ + /* TRANSLATORS: Metadata cache was not updated, likely because it was recent enough */ g_print ("%s\n", _("AppStream cache update is not necessary.")); - return 0; } + + return 0; } /** @@ -95,22 +92,22 @@ static AsPool* ascli_data_pool_new_and_open (const gchar *cachepath, gboolean no_cache, GError **error) { AsPool *dpool; + AsPoolFlags flags; dpool = as_pool_new (); - if (cachepath == NULL) { - /* no cache object to load, we can use a normal pool - unless (system) caching - * is generally disallowed. */ - if (no_cache) { - as_pool_set_cache_flags (dpool, AS_CACHE_FLAG_USE_USER); - } + flags = as_pool_get_flags (dpool); + if (no_cache) + as_flags_add (flags, AS_POOL_FLAG_IGNORE_CACHE_AGE); + if (cachepath == NULL) { + /* no cache object to load, we can use a normal pool */ as_pool_load (dpool, NULL, error); } else { /* use an exported cache object */ - as_pool_set_cache_flags (dpool, AS_CACHE_FLAG_USE_USER); - as_pool_set_cache_location (dpool, cachepath); + as_pool_override_cache_locations (dpool, cachepath, cachepath); as_pool_load (dpool, NULL, error); } + as_pool_set_flags (dpool, flags); return dpool; }