From bf73282c0b34bfc7772f952496b06150cceb5dae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Scharfe?= Date: Thu, 4 Oct 2018 17:09:06 +0200 Subject: [PATCH 1/5] fetch-pack: factor out is_unmatched_ref() Move the code to determine if a request is unmatched to its own little helper. This allows us to reuse it in a subsequent patch. Signed-off-by: Rene Scharfe Signed-off-by: Junio C Hamano --- fetch-pack.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/fetch-pack.c b/fetch-pack.c index 75047a4b2a..3b317952f0 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -543,6 +543,16 @@ static int tip_oids_contain(struct oidset *tip_oids, return oidset_contains(tip_oids, id); } +static int is_unmatched_ref(const struct ref *ref) +{ + struct object_id oid; + const char *p; + return ref->match_status == REF_NOT_MATCHED && + !parse_oid_hex(ref->name, &oid, &p) && + *p == '\0' && + oideq(&oid, &ref->old_oid); +} + static void filter_refs(struct fetch_pack_args *args, struct ref **refs, struct ref **sought, int nr_sought) @@ -591,15 +601,8 @@ static void filter_refs(struct fetch_pack_args *args, /* Append unmatched requests to the list */ for (i = 0; i < nr_sought; i++) { - struct object_id oid; - const char *p; - ref = sought[i]; - if (ref->match_status != REF_NOT_MATCHED) - continue; - if (parse_oid_hex(ref->name, &oid, &p) || - *p != '\0' || - !oideq(&oid, &ref->old_oid)) + if (!is_unmatched_ref(ref)) continue; if ((allow_unadvertised_object_request & From 22a164651114738c723c4459140e1e5330491467 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Scharfe?= Date: Thu, 4 Oct 2018 17:09:39 +0200 Subject: [PATCH 2/5] fetch-pack: load tip_oids eagerly iff needed tip_oids_contain() lazily loads refs into an oidset at its first call. It abuses the internal (sub)member .map.tablesize of that oidset to check if it has done that already. Determine if the oidset needs to be populated upfront and then do that instead. This duplicates a loop, but simplifies the existing one by separating concerns between the two. Signed-off-by: Rene Scharfe Signed-off-by: Junio C Hamano --- fetch-pack.c | 36 +++++++++++++++--------------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/fetch-pack.c b/fetch-pack.c index 3b317952f0..53914563b5 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -526,23 +526,6 @@ static void add_refs_to_oidset(struct oidset *oids, struct ref *refs) oidset_insert(oids, &refs->old_oid); } -static int tip_oids_contain(struct oidset *tip_oids, - struct ref *unmatched, struct ref *newlist, - const struct object_id *id) -{ - /* - * Note that this only looks at the ref lists the first time it's - * called. This works out in filter_refs() because even though it may - * add to "newlist" between calls, the additions will always be for - * oids that are already in the set. - */ - if (!tip_oids->map.map.tablesize) { - add_refs_to_oidset(tip_oids, unmatched); - add_refs_to_oidset(tip_oids, newlist); - } - return oidset_contains(tip_oids, id); -} - static int is_unmatched_ref(const struct ref *ref) { struct object_id oid; @@ -563,6 +546,8 @@ static void filter_refs(struct fetch_pack_args *args, struct ref *ref, *next; struct oidset tip_oids = OIDSET_INIT; int i; + int strict = !(allow_unadvertised_object_request & + (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1)); i = 0; for (ref = *refs; ref; ref = next) { @@ -599,16 +584,25 @@ static void filter_refs(struct fetch_pack_args *args, } } + if (strict) { + for (i = 0; i < nr_sought; i++) { + ref = sought[i]; + if (!is_unmatched_ref(ref)) + continue; + + add_refs_to_oidset(&tip_oids, unmatched); + add_refs_to_oidset(&tip_oids, newlist); + break; + } + } + /* Append unmatched requests to the list */ for (i = 0; i < nr_sought; i++) { ref = sought[i]; if (!is_unmatched_ref(ref)) continue; - if ((allow_unadvertised_object_request & - (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1)) || - tip_oids_contain(&tip_oids, unmatched, newlist, - &ref->old_oid)) { + if (!strict || oidset_contains(&tip_oids, &ref->old_oid)) { ref->match_status = REF_MATCHED; *newtail = copy_ref(ref); newtail = &(*newtail)->next; From 9249ca26aca3ae3f21f812593c7a5498736ae29a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Scharfe?= Date: Thu, 4 Oct 2018 17:10:54 +0200 Subject: [PATCH 3/5] khash: factor out kh_release_* Add a function for releasing the khash-internal allocations, but not the khash structure itself. It can be used with on-stack khash structs. Signed-off-by: Rene Scharfe Signed-off-by: Junio C Hamano --- khash.h | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/khash.h b/khash.h index 07b4cc2e67..d10caa0c35 100644 --- a/khash.h +++ b/khash.h @@ -82,11 +82,16 @@ static const double __ac_HASH_UPPER = 0.77; SCOPE kh_##name##_t *kh_init_##name(void) { \ return (kh_##name##_t*)xcalloc(1, sizeof(kh_##name##_t)); \ } \ + SCOPE void kh_release_##name(kh_##name##_t *h) \ + { \ + free(h->flags); \ + free((void *)h->keys); \ + free((void *)h->vals); \ + } \ SCOPE void kh_destroy_##name(kh_##name##_t *h) \ { \ if (h) { \ - free((void *)h->keys); free(h->flags); \ - free((void *)h->vals); \ + kh_release_##name(h); \ free(h); \ } \ } \ From 8b2f8cbcb16b1a9775214fe1d69aeb1580ae179d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Scharfe?= Date: Thu, 4 Oct 2018 17:13:06 +0200 Subject: [PATCH 4/5] oidset: use khash MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reimplement oidset using khash.h in order to reduce its memory footprint and make it faster. Performance of a command that mainly checks for duplicate objects using an oidset, with master and Clang 6.0.1: $ cmd="./git-cat-file --batch-all-objects --unordered --buffer --batch-check='%(objectname)'" $ /usr/bin/time $cmd >/dev/null 0.22user 0.03system 0:00.25elapsed 99%CPU (0avgtext+0avgdata 48484maxresident)k 0inputs+0outputs (0major+11204minor)pagefaults 0swaps $ hyperfine "$cmd" Benchmark #1: ./git-cat-file --batch-all-objects --unordered --buffer --batch-check='%(objectname)' Time (mean ± σ): 250.0 ms ± 6.0 ms [User: 225.9 ms, System: 23.6 ms] Range (min … max): 242.0 ms … 261.1 ms And with this patch: $ /usr/bin/time $cmd >/dev/null 0.14user 0.00system 0:00.15elapsed 100%CPU (0avgtext+0avgdata 41396maxresident)k 0inputs+0outputs (0major+8318minor)pagefaults 0swaps $ hyperfine "$cmd" Benchmark #1: ./git-cat-file --batch-all-objects --unordered --buffer --batch-check='%(objectname)' Time (mean ± σ): 151.9 ms ± 4.9 ms [User: 130.5 ms, System: 21.2 ms] Range (min … max): 148.2 ms … 170.4 ms Initial-patch-by: Jeff King Signed-off-by: Rene Scharfe Signed-off-by: Junio C Hamano --- oidset.c | 34 ++++++++++++---------------------- oidset.h | 36 ++++++++++++++++++++++++++++-------- 2 files changed, 40 insertions(+), 30 deletions(-) diff --git a/oidset.c b/oidset.c index 454c54f933..9836d427ef 100644 --- a/oidset.c +++ b/oidset.c @@ -3,38 +3,28 @@ int oidset_contains(const struct oidset *set, const struct object_id *oid) { - if (!set->map.map.tablesize) - return 0; - return !!oidmap_get(&set->map, oid); + khiter_t pos = kh_get_oid(&set->set, *oid); + return pos != kh_end(&set->set); } int oidset_insert(struct oidset *set, const struct object_id *oid) { - struct oidmap_entry *entry; - - if (!set->map.map.tablesize) - oidmap_init(&set->map, 0); - else if (oidset_contains(set, oid)) - return 1; - - entry = xmalloc(sizeof(*entry)); - oidcpy(&entry->oid, oid); - - oidmap_put(&set->map, entry); - return 0; + int added; + kh_put_oid(&set->set, *oid, &added); + return !added; } int oidset_remove(struct oidset *set, const struct object_id *oid) { - struct oidmap_entry *entry; - - entry = oidmap_remove(&set->map, oid); - free(entry); - - return (entry != NULL); + khiter_t pos = kh_get_oid(&set->set, *oid); + if (pos == kh_end(&set->set)) + return 0; + kh_del_oid(&set->set, pos); + return 1; } void oidset_clear(struct oidset *set) { - oidmap_free(&set->map, 1); + kh_release_oid(&set->set); + oidset_init(set, 0); } diff --git a/oidset.h b/oidset.h index 40ec5f87fe..4b90540cd4 100644 --- a/oidset.h +++ b/oidset.h @@ -1,7 +1,8 @@ #ifndef OIDSET_H #define OIDSET_H -#include "oidmap.h" +#include "hashmap.h" +#include "khash.h" /** * This API is similar to sha1-array, in that it maintains a set of object ids @@ -15,19 +16,33 @@ * table overhead. */ +static inline unsigned int oid_hash(struct object_id oid) +{ + return sha1hash(oid.hash); +} + +static inline int oid_equal(struct object_id a, struct object_id b) +{ + return oideq(&a, &b); +} + +KHASH_INIT(oid, struct object_id, int, 0, oid_hash, oid_equal) + /** * A single oidset; should be zero-initialized (or use OIDSET_INIT). */ struct oidset { - struct oidmap map; + kh_oid_t set; }; -#define OIDSET_INIT { OIDMAP_INIT } +#define OIDSET_INIT { { 0 } } static inline void oidset_init(struct oidset *set, size_t initial_size) { - oidmap_init(&set->map, initial_size); + memset(&set->set, 0, sizeof(set->set)); + if (initial_size) + kh_resize_oid(&set->set, initial_size); } /** @@ -58,19 +73,24 @@ int oidset_remove(struct oidset *set, const struct object_id *oid); void oidset_clear(struct oidset *set); struct oidset_iter { - struct oidmap_iter m_iter; + kh_oid_t *set; + khiter_t iter; }; static inline void oidset_iter_init(struct oidset *set, struct oidset_iter *iter) { - oidmap_iter_init(&set->map, &iter->m_iter); + iter->set = &set->set; + iter->iter = kh_begin(iter->set); } static inline struct object_id *oidset_iter_next(struct oidset_iter *iter) { - struct oidmap_entry *e = oidmap_iter_next(&iter->m_iter); - return e ? &e->oid : NULL; + for (; iter->iter != kh_end(iter->set); iter->iter++) { + if (kh_exist(iter->set, iter->iter)) + return &kh_key(iter->set, iter->iter++); + } + return NULL; } static inline struct object_id *oidset_iter_first(struct oidset *set, From 8c84ae659e0e17d55f5ddc58bc79855ed7650e00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Scharfe?= Date: Thu, 4 Oct 2018 17:14:37 +0200 Subject: [PATCH 5/5] oidset: uninline oidset_init() There is no need to inline oidset_init(), as it's typically only called twice in the lifetime of an oidset (once at the beginning and at the end by oidset_clear()) and kh_resize_* is quite big, so move its definition to oidset.c. Document it while we're at it. Signed-off-by: Rene Scharfe Signed-off-by: Junio C Hamano --- oidset.c | 7 +++++++ oidset.h | 13 +++++++------ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/oidset.c b/oidset.c index 9836d427ef..fe4eb921df 100644 --- a/oidset.c +++ b/oidset.c @@ -1,6 +1,13 @@ #include "cache.h" #include "oidset.h" +void oidset_init(struct oidset *set, size_t initial_size) +{ + memset(&set->set, 0, sizeof(set->set)); + if (initial_size) + kh_resize_oid(&set->set, initial_size); +} + int oidset_contains(const struct oidset *set, const struct object_id *oid) { khiter_t pos = kh_get_oid(&set->set, *oid); diff --git a/oidset.h b/oidset.h index 4b90540cd4..c9d0f6d3cc 100644 --- a/oidset.h +++ b/oidset.h @@ -38,12 +38,13 @@ struct oidset { #define OIDSET_INIT { { 0 } } -static inline void oidset_init(struct oidset *set, size_t initial_size) -{ - memset(&set->set, 0, sizeof(set->set)); - if (initial_size) - kh_resize_oid(&set->set, initial_size); -} +/** + * Initialize the oidset structure `set`. + * + * If `initial_size` is bigger than 0 then preallocate to allow inserting + * the specified number of elements without further allocations. + */ +void oidset_init(struct oidset *set, size_t initial_size); /** * Returns true iff `set` contains `oid`.