[PATCH 00/26] nd/shallow-deepen updates

classic Classic list List threaded Threaded
32 messages Options
12
Reply | Threaded
Open this post in threaded view
|

[PATCH 20/26] upload-pack: support define shallow boundary by excluding revisions

Duy Nguyen
This should allow the user to say "create a shallow clone of this branch
after version <some-tag>".

Short refs are accepted and expanded at the server side with expand_ref()
because we cannot expand (unknown) refs from the client side.

Like deepen-since, deepen-not cannot be used with deepen. But deepen-not
can be mixed with deepen-since. The result is exactly how you do the
command "git rev-list --since=... --not ref".

Signed-off-by: Nguyễn Thái Ngọc Duy <[hidden email]>
---
 Documentation/technical/pack-protocol.txt         |  3 ++-
 Documentation/technical/protocol-capabilities.txt |  9 +++++++++
 upload-pack.c                                     | 23 +++++++++++++++++++++--
 3 files changed, 32 insertions(+), 3 deletions(-)

diff --git a/Documentation/technical/pack-protocol.txt b/Documentation/technical/pack-protocol.txt
index 9251df1..dee33a6 100644
--- a/Documentation/technical/pack-protocol.txt
+++ b/Documentation/technical/pack-protocol.txt
@@ -220,7 +220,8 @@ out of what the server said it could do with the first 'want' line.
   shallow-line      =  PKT-LINE("shallow" SP obj-id)
 
   depth-request     =  PKT-LINE("deepen" SP depth) /
-       PKT-LINE("deepen-since" SP timestamp)
+       PKT-LINE("deepen-since" SP timestamp) /
+       PKT-LINE("deepen-not" SP ref)
 
   first-want        =  PKT-LINE("want" SP obj-id SP capability-list)
   additional-want   =  PKT-LINE("want" SP obj-id)
diff --git a/Documentation/technical/protocol-capabilities.txt b/Documentation/technical/protocol-capabilities.txt
index f08cc4e..0e6b57d 100644
--- a/Documentation/technical/protocol-capabilities.txt
+++ b/Documentation/technical/protocol-capabilities.txt
@@ -188,6 +188,15 @@ specific time, instead of depth. Internally it's equivalent of doing
 "rev-list --max-age=<timestamp>" on the server side. "deepen-since"
 cannot be used with "deepen".
 
+deepen-not
+----------
+
+This capability adds "deepen-not" command to fetch-pack/upload-pack
+protocol so the client can request shallow clones that are cut at a
+specific revision, instead of depth. Internally it's equivalent of
+doing "rev-list --not <rev>" on the server side. "deepen-not"
+cannot be used with "deepen", but can be used with "deepen-since".
+
 no-progress
 -----------
 
diff --git a/upload-pack.c b/upload-pack.c
index 58c0936..95a0bfb 100644
--- a/upload-pack.c
+++ b/upload-pack.c
@@ -628,6 +628,7 @@ static void deepen_by_rev_list(int ac, const char **av,
 static void receive_needs(void)
 {
  struct object_array shallows = OBJECT_ARRAY_INIT;
+ struct string_list deepen_not = STRING_LIST_INIT_DUP;
  int depth = 0;
  int has_non_tip = 0;
  unsigned long deepen_since = 0;
@@ -678,6 +679,16 @@ static void receive_needs(void)
  deepen_rev_list = 1;
  continue;
  }
+ if (skip_prefix(line, "deepen-not ", &arg)) {
+ char *ref = NULL;
+ unsigned char sha1[20];
+ if (expand_ref(arg, strlen(arg), sha1, &ref) != 1)
+ die("git upload-pack: ambiguous deepen-not: %s", line);
+ string_list_append(&deepen_not, ref);
+ free(ref);
+ deepen_rev_list = 1;
+ continue;
+ }
  if (!skip_prefix(line, "want ", &arg) ||
     get_sha1_hex(arg, sha1_buf))
  die("git upload-pack: protocol error, "
@@ -732,7 +743,7 @@ static void receive_needs(void)
  if (depth == 0 && !deepen_rev_list && shallows.nr == 0)
  return;
  if (depth > 0 && deepen_rev_list)
- die("git upload-pack: deepen and deepen-since cannot be used together");
+ die("git upload-pack: deepen and deepen-since (or deepen-not) cannot be used together");
  if (depth > 0)
  deepen(depth, &shallows);
  else if (deepen_rev_list) {
@@ -742,6 +753,14 @@ static void receive_needs(void)
  argv_array_push(&av, "rev-list");
  if (deepen_since)
  argv_array_pushf(&av, "--max-age=%lu", deepen_since);
+ if (deepen_not.nr) {
+ argv_array_push(&av, "--not");
+ for (i = 0; i < deepen_not.nr; i++) {
+ struct string_list_item *s = deepen_not.items + i;
+ argv_array_push(&av, s->string);
+ }
+ argv_array_push(&av, "--not");
+ }
  for (i = 0; i < want_obj.nr; i++) {
  struct object *o = want_obj.objects[i].item;
  argv_array_push(&av, oid_to_hex(&o->oid));
@@ -797,7 +816,7 @@ static int send_ref(const char *refname, const struct object_id *oid,
     int flag, void *cb_data)
 {
  static const char *capabilities = "multi_ack thin-pack side-band"
- " side-band-64k ofs-delta shallow deepen-since no-progress"
+ " side-band-64k ofs-delta shallow deepen-since deepen-not no-progress"
  " include-tag multi_ack_detailed";
  const char *refname_nons = strip_namespace(refname);
  struct object_id peeled;
--
2.8.0.rc0.210.gd302cd2

--
To unsubscribe from this list: send the line "unsubscribe git" in
the body of a message to [hidden email]
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Reply | Threaded
Open this post in threaded view
|

[PATCH 21/26] fetch: define shallow boundary with --shallow-exclude

Duy Nguyen
In reply to this post by Duy Nguyen
Signed-off-by: Nguyễn Thái Ngọc Duy <[hidden email]>
---
 Documentation/fetch-options.txt     |  5 +++++
 Documentation/git-fetch-pack.txt    |  5 +++++
 Documentation/gitremote-helpers.txt |  4 ++++
 builtin/fetch-pack.c                |  7 +++++++
 builtin/fetch.c                     | 21 ++++++++++++++++++---
 fetch-pack.c                        | 15 ++++++++++++++-
 fetch-pack.h                        |  1 +
 remote-curl.c                       |  9 +++++++++
 transport-helper.c                  | 24 ++++++++++++++++++++++++
 transport.c                         |  4 ++++
 transport.h                         |  6 ++++++
 11 files changed, 97 insertions(+), 4 deletions(-)

diff --git a/Documentation/fetch-options.txt b/Documentation/fetch-options.txt
index 8738d3d..7aa1285 100644
--- a/Documentation/fetch-options.txt
+++ b/Documentation/fetch-options.txt
@@ -18,6 +18,11 @@
  Deepen or shorten the history of a shallow repository to
  include all reachable commits after <date>.
 
+--shallow-exclude=<revision>::
+ Deepen or shorten the history of a shallow repository to
+ exclude commits reachable from a specified remote branch or tag.
+ This option can be specified multiple times.
+
 --unshallow::
  If the source repository is complete, convert a shallow
  repository to a complete one, removing all the limitations
diff --git a/Documentation/git-fetch-pack.txt b/Documentation/git-fetch-pack.txt
index 99e6257..4d15b04 100644
--- a/Documentation/git-fetch-pack.txt
+++ b/Documentation/git-fetch-pack.txt
@@ -91,6 +91,11 @@ be in a separate packet, and the list must end with a flush packet.
  Deepen or shorten the history of a shallow'repository to
  include all reachable commits after <date>.
 
+--shallow-exclude=<revision>::
+ Deepen or shorten the history of a shallow repository to
+ exclude commits reachable from a specified remote branch or tag.
+ This option can be specified multiple times.
+
 --no-progress::
  Do not show the progress.
 
diff --git a/Documentation/gitremote-helpers.txt b/Documentation/gitremote-helpers.txt
index 9971d9a..75bb638 100644
--- a/Documentation/gitremote-helpers.txt
+++ b/Documentation/gitremote-helpers.txt
@@ -418,6 +418,10 @@ set by Git if the remote helper has the 'option' capability.
 'option deepen-since <timestamp>::
  Deepens the history of a shallow repository based on time.
 
+'option deepen-not <ref>::
+ Deepens the history of a shallow repository excluding ref.
+ Multiple options add up.
+
 'option followtags' {'true'|'false'}::
  If enabled the helper should automatically fetch annotated
  tag objects if the object the tag points at was transferred
diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c
index 0402e27..07570be 100644
--- a/builtin/fetch-pack.c
+++ b/builtin/fetch-pack.c
@@ -50,6 +50,7 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
  struct child_process *conn;
  struct fetch_pack_args args;
  struct sha1_array shallow = SHA1_ARRAY_INIT;
+ struct string_list deepen_not = STRING_LIST_INIT_DUP;
 
  packet_trace_identity("fetch-pack");
 
@@ -108,6 +109,10 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
  args.deepen_since = xstrdup(arg);
  continue;
  }
+ if (skip_prefix(arg, "--shallow-exclude=", &arg)) {
+ string_list_append(&deepen_not, arg);
+ continue;
+ }
  if (!strcmp("--no-progress", arg)) {
  args.no_progress = 1;
  continue;
@@ -135,6 +140,8 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
  }
  usage(fetch_pack_usage);
  }
+ if (deepen_not.nr)
+ args.deepen_not = &deepen_not;
 
  if (i < argc)
  dest = argv[i++];
diff --git a/builtin/fetch.c b/builtin/fetch.c
index 283aa95..154b9ad 100644
--- a/builtin/fetch.c
+++ b/builtin/fetch.c
@@ -41,6 +41,7 @@ static int max_children = 1;
 static const char *depth;
 static const char *deepen_since;
 static const char *upload_pack;
+static struct string_list deepen_not = STRING_LIST_INIT_NODUP;
 static struct strbuf default_rla = STRBUF_INIT;
 static struct transport *gtransport;
 static struct transport *gsecondary;
@@ -50,6 +51,13 @@ static int shown_url = 0;
 static int refmap_alloc, refmap_nr;
 static const char **refmap_array;
 
+static int option_parse_deepen_not(const struct option *opt,
+   const char *arg, int unset)
+{
+ string_list_append(&deepen_not, arg);
+ return 0;
+}
+
 static int option_parse_recurse_submodules(const struct option *opt,
    const char *arg, int unset)
 {
@@ -118,6 +126,9 @@ static struct option builtin_fetch_options[] = {
    N_("deepen history of shallow clone")),
  OPT_STRING(0, "shallow-since", &deepen_since, N_("time"),
    N_("deepen history of shallow repository based on time")),
+ { OPTION_CALLBACK, 0, "shallow-exclude", NULL, N_("revision"),
+    N_("deepen history of shallow clone by excluding rev"),
+    PARSE_OPT_NONEG, option_parse_deepen_not },
  { OPTION_SET_INT, 0, "unshallow", &unshallow, NULL,
    N_("convert to a complete repository"),
    PARSE_OPT_NONEG | PARSE_OPT_NOARG, NULL, 1 },
@@ -875,6 +886,9 @@ static struct transport *prepare_transport(struct remote *remote, int deepen)
  set_option(transport, TRANS_OPT_DEPTH, depth);
  if (deepen && deepen_since)
  set_option(transport, TRANS_OPT_DEEPEN_SINCE, deepen_since);
+ if (deepen && deepen_not.nr)
+ set_option(transport, TRANS_OPT_DEEPEN_NOT,
+   (const char *)&deepen_not);
  if (update_shallow)
  set_option(transport, TRANS_OPT_UPDATE_SHALLOW, "yes");
  return transport;
@@ -889,9 +903,10 @@ static void backfill_tags(struct transport *transport, struct ref *ref_map)
  * when remote helper is used (setting it to an empty string
  * is not unsetting). We could extend the remote helper
  * protocol for that, but for now, just force a new connection
- * without deepen-since.
+ * without deepen-since. Similar story for deepen-not.
  */
- cannot_reuse = transport->cannot_reuse || deepen_since;
+ cannot_reuse = transport->cannot_reuse ||
+ deepen_since || deepen_not.nr;
  if (cannot_reuse) {
  gsecondary = prepare_transport(transport->remote, 0);
  transport = gsecondary;
@@ -1182,7 +1197,7 @@ int cmd_fetch(int argc, const char **argv, const char *prefix)
  /* no need to be strict, transport_set_option() will validate it again */
  if (depth && atoi(depth) < 1)
  die(_("depth %s is not a positive number"), depth);
- if (depth || deepen_since)
+ if (depth || deepen_since || deepen_not.nr)
  deepen = 1;
 
  if (recurse_submodules != RECURSE_SUBMODULES_OFF) {
diff --git a/fetch-pack.c b/fetch-pack.c
index a2f25c1..ad7d00f 100644
--- a/fetch-pack.c
+++ b/fetch-pack.c
@@ -22,6 +22,7 @@ static int unpack_limit = 100;
 static int prefer_ofs_delta = 1;
 static int no_done;
 static int deepen_since_ok;
+static int deepen_not_ok;
 static int fetch_fsck_objects = -1;
 static int transfer_fsck_objects = -1;
 static int agent_supported;
@@ -328,6 +329,7 @@ static int find_common(struct fetch_pack_args *args,
  if (args->include_tag)   strbuf_addstr(&c, " include-tag");
  if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
  if (deepen_since_ok)    strbuf_addstr(&c, " deepen-since");
+ if (deepen_not_ok)      strbuf_addstr(&c, " deepen-not");
  if (agent_supported)    strbuf_addf(&c, " agent=%s",
     git_user_agent_sanitized());
  packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
@@ -351,6 +353,13 @@ static int find_common(struct fetch_pack_args *args,
  unsigned long max_age = approxidate(args->deepen_since);
  packet_buf_write(&req_buf, "deepen-since %lu", max_age);
  }
+ if (args->deepen_not) {
+ int i;
+ for (i = 0; i < args->deepen_not->nr; i++) {
+ struct string_list_item *s = args->deepen_not->items + i;
+ packet_buf_write(&req_buf, "deepen-not %s", s->string);
+ }
+ }
  packet_buf_flush(&req_buf);
  state_len = req_buf.len;
 
@@ -818,7 +827,7 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 
  if ((args->depth > 0 || is_repository_shallow()) && !server_supports("shallow"))
  die(_("Server does not support shallow clients"));
- if (args->depth > 0 || args->deepen_since)
+ if (args->depth > 0 || args->deepen_since || args->deepen_not)
  args->deepen = 1;
  if (server_supports("multi_ack_detailed")) {
  print_verbose(args, _("Server supports multi_ack_detailed"));
@@ -870,6 +879,10 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args,
  deepen_since_ok = 1;
  else if (args->deepen_since)
  die(_("Server does not support --shallow-since"));
+ if (server_supports("deepen-not"))
+ deepen_not_ok = 1;
+ else if (args->deepen_not)
+ die(_("Server does not support --shallow-exclude"));
 
  if (everything_local(args, &ref, sought, nr_sought)) {
  packet_flush(fd[1]);
diff --git a/fetch-pack.h b/fetch-pack.h
index f7eadb2..144301f 100644
--- a/fetch-pack.h
+++ b/fetch-pack.h
@@ -11,6 +11,7 @@ struct fetch_pack_args {
  int unpacklimit;
  int depth;
  const char *deepen_since;
+ const struct string_list *deepen_not;
  unsigned quiet:1;
  unsigned keep_pack:1;
  unsigned lock_pack:1;
diff --git a/remote-curl.c b/remote-curl.c
index ffa4faa..71122ac 100644
--- a/remote-curl.c
+++ b/remote-curl.c
@@ -21,6 +21,7 @@ struct options {
  int verbosity;
  unsigned long depth;
  char *deepen_since;
+ struct string_list deepen_not;
  unsigned progress : 1,
  check_self_contained_and_connected : 1,
  cloning : 1,
@@ -65,6 +66,10 @@ static int set_option(const char *name, const char *value)
  options.deepen_since = xstrdup(value);
  return 0;
  }
+ else if (!strcmp(name, "deepen-not")) {
+ string_list_append(&options.deepen_not, value);
+ return 0;
+ }
  else if (!strcmp(name, "followtags")) {
  if (!strcmp(value, "true"))
  options.followtags = 1;
@@ -756,6 +761,9 @@ static int fetch_git(struct discovery *heads,
  argv_array_pushf(&args, "--depth=%lu", options.depth);
  if (options.deepen_since)
  argv_array_pushf(&args, "--shallow-since=%s", options.deepen_since);
+ for (i = 0; i < options.deepen_not.nr; i++)
+ argv_array_pushf(&args, "--shallow-exclude=%s",
+ options.deepen_not.items[i].string);
  argv_array_push(&args, url.buf);
 
  for (i = 0; i < nr_heads; i++) {
@@ -977,6 +985,7 @@ int main(int argc, const char **argv)
  options.verbosity = 1;
  options.progress = !!isatty(2);
  options.thin = 1;
+ string_list_init(&options.deepen_not, 1);
 
  remote = remote_get(argv[1]);
 
diff --git a/transport-helper.c b/transport-helper.c
index 35023da..d644568 100644
--- a/transport-helper.c
+++ b/transport-helper.c
@@ -282,6 +282,26 @@ static int strbuf_set_helper_option(struct helper_data *data,
  return ret;
 }
 
+static int string_list_set_helper_option(struct helper_data *data,
+ const char *name,
+ struct string_list *list)
+{
+ struct strbuf buf = STRBUF_INIT;
+ int i, ret = 0;
+
+ for (i = 0; i < list->nr; i++) {
+ strbuf_addf(&buf, "option %s ", name);
+ quote_c_style(list->items[i].string, &buf, NULL, 0);
+ strbuf_addch(&buf, '\n');
+
+ if ((ret = strbuf_set_helper_option(data, &buf)))
+ break;
+ strbuf_reset(&buf);
+ }
+ strbuf_release(&buf);
+ return ret;
+}
+
 static int set_helper_option(struct transport *transport,
   const char *name, const char *value)
 {
@@ -294,6 +314,10 @@ static int set_helper_option(struct transport *transport,
  if (!data->option)
  return 1;
 
+ if (!strcmp(name, "deepen-not"))
+ return string_list_set_helper_option(data, name,
+     (struct string_list *)value);
+
  for (i = 0; i < ARRAY_SIZE(unsupported_options); i++) {
  if (!strcmp(name, unsupported_options[i]))
  return 1;
diff --git a/transport.c b/transport.c
index f04a302..3e6f3aa 100644
--- a/transport.c
+++ b/transport.c
@@ -154,6 +154,9 @@ static int set_git_option(struct git_transport_options *opts,
  } else if (!strcmp(name, TRANS_OPT_DEEPEN_SINCE)) {
  opts->deepen_since = value;
  return 0;
+ } else if (!strcmp(name, TRANS_OPT_DEEPEN_NOT)) {
+ opts->deepen_not = (const struct string_list *)value;
+ return 0;
  }
  return 1;
 }
@@ -209,6 +212,7 @@ static int fetch_refs_via_pack(struct transport *transport,
  args.no_progress = !transport->progress;
  args.depth = data->options.depth;
  args.deepen_since = data->options.deepen_since;
+ args.deepen_not = data->options.deepen_not;
  args.check_self_contained_and_connected =
  data->options.check_self_contained_and_connected;
  args.cloning = transport->cloning;
diff --git a/transport.h b/transport.h
index 9c10a44..ab61932 100644
--- a/transport.h
+++ b/transport.h
@@ -5,6 +5,8 @@
 #include "run-command.h"
 #include "remote.h"
 
+struct string_list;
+
 struct git_transport_options {
  unsigned thin : 1;
  unsigned keep : 1;
@@ -14,6 +16,7 @@ struct git_transport_options {
  unsigned update_shallow : 1;
  int depth;
  const char *deepen_since;
+ const struct string_list *deepen_not;
  const char *uploadpack;
  const char *receivepack;
  struct push_cas_option *cas;
@@ -175,6 +178,9 @@ int transport_restrict_protocols(void);
 /* Limit the depth of the fetch based on time if not null */
 #define TRANS_OPT_DEEPEN_SINCE "deepen-since"
 
+/* Limit the depth of the fetch based on revs if not null */
+#define TRANS_OPT_DEEPEN_NOT "deepen-not"
+
 /* Aggressively fetch annotated tags if possible */
 #define TRANS_OPT_FOLLOWTAGS "followtags"
 
--
2.8.0.rc0.210.gd302cd2

--
To unsubscribe from this list: send the line "unsubscribe git" in
the body of a message to [hidden email]
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Reply | Threaded
Open this post in threaded view
|

[PATCH 22/26] clone: define shallow clone boundary with --shallow-exclude

Duy Nguyen
In reply to this post by Duy Nguyen
Signed-off-by: Nguyễn Thái Ngọc Duy <[hidden email]>
---
 Documentation/git-clone.txt |  5 +++++
 builtin/clone.c             | 18 +++++++++++++++++-
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/Documentation/git-clone.txt b/Documentation/git-clone.txt
index a410409..5049663 100644
--- a/Documentation/git-clone.txt
+++ b/Documentation/git-clone.txt
@@ -196,6 +196,11 @@ objects from the source repository into a pack in the cloned repository.
 --shallow-since=<date>::
  Create a shallow clone with a history after the specified time.
 
+--shallow-exclude=<revision>::
+ Create a shallow clone with a history, excluding commits
+ reachable from a specified remote branch or tag.  This option
+ can be specified multiple times.
+
 --[no-]single-branch::
  Clone only the history leading to the tip of a single branch,
  either specified by the `--branch` option or the primary
diff --git a/builtin/clone.c b/builtin/clone.c
index dc2ef4f..5ccf6b7 100644
--- a/builtin/clone.c
+++ b/builtin/clone.c
@@ -44,6 +44,7 @@ static int deepen;
 static char *option_template, *option_depth, *option_since;
 static char *option_origin = NULL;
 static char *option_branch = NULL;
+static struct string_list option_not = STRING_LIST_INIT_NODUP;
 static const char *real_git_dir;
 static char *option_upload_pack = "git-upload-pack";
 static int option_verbosity;
@@ -52,6 +53,13 @@ static struct string_list option_config;
 static struct string_list option_reference;
 static int option_dissociate;
 
+static int option_parse_deepen_not(const struct option *opt,
+   const char *arg, int unset)
+{
+ string_list_append(&option_not, arg);
+ return 0;
+}
+
 static struct option builtin_clone_options[] = {
  OPT__VERBOSITY(&option_verbosity),
  OPT_BOOL(0, "progress", &option_progress,
@@ -89,6 +97,9 @@ static struct option builtin_clone_options[] = {
     N_("create a shallow clone of that depth")),
  OPT_STRING(0, "shallow-since", &option_since, N_("time"),
     N_("create a shallow clone since a specific time")),
+ { OPTION_CALLBACK, 0, "shallow-exclude", NULL, N_("revision"),
+    N_("deepen history of shallow clone by excluding rev"),
+    PARSE_OPT_NONEG, option_parse_deepen_not },
  OPT_BOOL(0, "single-branch", &option_single_branch,
     N_("clone only one branch, HEAD or --branch")),
  OPT_STRING(0, "separate-git-dir", &real_git_dir, N_("gitdir"),
@@ -852,7 +863,7 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
  usage_msg_opt(_("You must specify a repository to clone."),
  builtin_clone_usage, builtin_clone_options);
 
- if (option_depth || option_since)
+ if (option_depth || option_since || option_not.nr)
  deepen = 1;
  if (option_single_branch == -1)
  option_single_branch = deepen ? 1 : 0;
@@ -983,6 +994,8 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
  warning(_("--depth is ignored in local clones; use file:// instead."));
  if (option_since)
  warning(_("--shallow-since is ignored in local clones; use file:// instead."));
+ if (option_not.nr)
+ warning(_("--shallow-exclude is ignored in local clones; use file:// instead."));
  if (!access(mkpath("%s/shallow", path), F_OK)) {
  if (option_local > 0)
  warning(_("source repository is shallow, ignoring --local"));
@@ -1004,6 +1017,9 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
  if (option_since)
  transport_set_option(transport, TRANS_OPT_DEEPEN_SINCE,
      option_since);
+ if (option_not.nr)
+ transport_set_option(transport, TRANS_OPT_DEEPEN_NOT,
+     (const char *)&option_not);
  if (option_single_branch)
  transport_set_option(transport, TRANS_OPT_FOLLOWTAGS, "1");
 
--
2.8.0.rc0.210.gd302cd2

--
To unsubscribe from this list: send the line "unsubscribe git" in
the body of a message to [hidden email]
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Reply | Threaded
Open this post in threaded view
|

[PATCH 23/26] t5500, t5539: tests for shallow depth excluding a ref

Duy Nguyen
In reply to this post by Duy Nguyen
Signed-off-by: Nguyễn Thái Ngọc Duy <[hidden email]>
---
 t/t5500-fetch-pack.sh         | 22 ++++++++++++++++++++++
 t/t5539-fetch-http-shallow.sh | 22 ++++++++++++++++++++++
 2 files changed, 44 insertions(+)

diff --git a/t/t5500-fetch-pack.sh b/t/t5500-fetch-pack.sh
index 26f050d..a3fe5ca 100755
--- a/t/t5500-fetch-pack.sh
+++ b/t/t5500-fetch-pack.sh
@@ -661,4 +661,26 @@ test_expect_success 'fetch shallow since ...' '
  test_cmp expected actual
 '
 
+test_expect_success 'shallow clone exclude tag two' '
+ test_create_repo shallow-exclude &&
+ (
+ cd shallow-exclude &&
+ test_commit one &&
+ test_commit two &&
+ test_commit three &&
+ git clone --shallow-exclude two "file://$(pwd)/." ../shallow12 &&
+ git -C ../shallow12 log --pretty=tformat:%s HEAD >actual &&
+ echo three >expected &&
+ test_cmp expected actual
+ )
+'
+
+test_expect_success 'fetch exclude tag one' '
+ git -C shallow12 fetch --shallow-exclude one origin &&
+ git -C shallow12 log --pretty=tformat:%s origin/master >actual &&
+ echo three >expected &&
+ echo two  >>expected &&
+ test_cmp expected actual
+'
+
 test_done
diff --git a/t/t5539-fetch-http-shallow.sh b/t/t5539-fetch-http-shallow.sh
index 6d77ca7..f71573d 100755
--- a/t/t5539-fetch-http-shallow.sh
+++ b/t/t5539-fetch-http-shallow.sh
@@ -98,6 +98,28 @@ test_expect_success 'fetch shallow since ...' '
  test_cmp expected actual
 '
 
+test_expect_success 'shallow clone exclude tag two' '
+ test_create_repo shallow-exclude &&
+ (
+ cd shallow-exclude &&
+ test_commit one &&
+ test_commit two &&
+ test_commit three &&
+ mv .git "$HTTPD_DOCUMENT_ROOT_PATH/shallow-exclude.git" &&
+ git clone --shallow-exclude two $HTTPD_URL/smart/shallow-exclude.git ../shallow12 &&
+ git -C ../shallow12 log --pretty=tformat:%s HEAD >actual &&
+ echo three >expected &&
+ test_cmp expected actual
+ )
+'
+
+test_expect_success 'fetch exclude tag one' '
+ git -C shallow12 fetch --shallow-exclude one origin &&
+ git -C shallow12 log --pretty=tformat:%s origin/master >actual &&
+ echo three >expected &&
+ echo two  >>expected &&
+ test_cmp expected actual
+'
 
 stop_httpd
 test_done
--
2.8.0.rc0.210.gd302cd2

--
To unsubscribe from this list: send the line "unsubscribe git" in
the body of a message to [hidden email]
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Reply | Threaded
Open this post in threaded view
|

[PATCH 24/26] upload-pack: split check_unreachable() in two, prep for get_reachable_list()

Duy Nguyen
In reply to this post by Duy Nguyen
Signed-off-by: Nguyễn Thái Ngọc Duy <[hidden email]>
---
 upload-pack.c | 41 ++++++++++++++++++++++++++---------------
 1 file changed, 26 insertions(+), 15 deletions(-)

diff --git a/upload-pack.c b/upload-pack.c
index 95a0bfb..9e4a4fa 100644
--- a/upload-pack.c
+++ b/upload-pack.c
@@ -452,24 +452,24 @@ static int is_our_ref(struct object *o)
  return o->flags & ((allow_hidden_ref ? HIDDEN_REF : 0) | OUR_REF);
 }
 
-static int check_unreachable(struct object_array *src)
+static int do_reachable_revlist(struct child_process *cmd,
+ struct object_array *src)
 {
  static const char *argv[] = {
  "rev-list", "--stdin", NULL,
  };
- static struct child_process cmd = CHILD_PROCESS_INIT;
  struct object *o;
  char namebuf[42]; /* ^ + SHA-1 + LF */
  int i;
 
- cmd.argv = argv;
- cmd.git_cmd = 1;
- cmd.no_stderr = 1;
- cmd.in = -1;
- cmd.out = -1;
+ cmd->argv = argv;
+ cmd->git_cmd = 1;
+ cmd->no_stderr = 1;
+ cmd->in = -1;
+ cmd->out = -1;
 
- if (start_command(&cmd))
- return 0;
+ if (start_command(cmd))
+ return -1;
 
  /*
  * If rev-list --stdin encounters an unknown commit, it
@@ -487,8 +487,8 @@ static int check_unreachable(struct object_array *src)
  if (!is_our_ref(o))
  continue;
  memcpy(namebuf + 1, oid_to_hex(&o->oid), GIT_SHA1_HEXSZ);
- if (write_in_full(cmd.in, namebuf, 42) < 0)
- return 0;
+ if (write_in_full(cmd->in, namebuf, 42) < 0)
+ return -1;
  }
  namebuf[40] = '\n';
  for (i = 0; i < src->nr; i++) {
@@ -496,18 +496,29 @@ static int check_unreachable(struct object_array *src)
  if (is_our_ref(o))
  continue;
  memcpy(namebuf, oid_to_hex(&o->oid), GIT_SHA1_HEXSZ);
- if (write_in_full(cmd.in, namebuf, 41) < 0)
- return 0;
+ if (write_in_full(cmd->in, namebuf, 41) < 0)
+ return -1;
  }
- close(cmd.in);
+ close(cmd->in);
 
  sigchain_pop(SIGPIPE);
+ return 0;
+}
+
+static int check_unreachable(struct object_array *src)
+{
+ struct child_process cmd = CHILD_PROCESS_INIT;
+ int i, ret = do_reachable_revlist(&cmd, src);
+ char buf[1];
+
+ if (ret < 0)
+ return 0;
 
  /*
  * The commits out of the rev-list are not ancestors of
  * our ref.
  */
- i = read_in_full(cmd.out, namebuf, 1);
+ i = read_in_full(cmd.out, buf, 1);
  if (i)
  return 0;
  close(cmd.out);
--
2.8.0.rc0.210.gd302cd2

--
To unsubscribe from this list: send the line "unsubscribe git" in
the body of a message to [hidden email]
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Reply | Threaded
Open this post in threaded view
|

[PATCH 25/26] upload-pack: add get_reachable_list()

Duy Nguyen
In reply to this post by Duy Nguyen
Signed-off-by: Nguyễn Thái Ngọc Duy <[hidden email]>
---
 object.h      |  2 +-
 upload-pack.c | 52 +++++++++++++++++++++++++++++++++++++++++++++++++---
 2 files changed, 50 insertions(+), 4 deletions(-)

diff --git a/object.h b/object.h
index f8b6442..614a006 100644
--- a/object.h
+++ b/object.h
@@ -31,7 +31,7 @@ struct object_array {
  * revision.h:      0---------10                                26
  * fetch-pack.c:    0---4
  * walker.c:        0-2
- * upload-pack.c:               11----------------19
+ * upload-pack.c:       4       11----------------19
  * builtin/blame.c:               12-13
  * bisect.c:                               16
  * bundle.c:                               16
diff --git a/upload-pack.c b/upload-pack.c
index 9e4a4fa..9bd590c 100644
--- a/upload-pack.c
+++ b/upload-pack.c
@@ -453,7 +453,8 @@ static int is_our_ref(struct object *o)
 }
 
 static int do_reachable_revlist(struct child_process *cmd,
- struct object_array *src)
+ struct object_array *src,
+ struct object_array *reachable)
 {
  static const char *argv[] = {
  "rev-list", "--stdin", NULL,
@@ -484,6 +485,8 @@ static int do_reachable_revlist(struct child_process *cmd,
  o = get_indexed_object(--i);
  if (!o)
  continue;
+ if (reachable && o->type == OBJ_COMMIT)
+ o->flags &= ~TMP_MARK;
  if (!is_our_ref(o))
  continue;
  memcpy(namebuf + 1, oid_to_hex(&o->oid), GIT_SHA1_HEXSZ);
@@ -493,8 +496,13 @@ static int do_reachable_revlist(struct child_process *cmd,
  namebuf[40] = '\n';
  for (i = 0; i < src->nr; i++) {
  o = src->objects[i].item;
- if (is_our_ref(o))
+ if (is_our_ref(o)) {
+ if (reachable)
+ add_object_array(o, NULL, reachable);
  continue;
+ }
+ if (reachable && o->type == OBJ_COMMIT)
+ o->flags |= TMP_MARK;
  memcpy(namebuf, oid_to_hex(&o->oid), GIT_SHA1_HEXSZ);
  if (write_in_full(cmd->in, namebuf, 41) < 0)
  return -1;
@@ -505,10 +513,48 @@ static int do_reachable_revlist(struct child_process *cmd,
  return 0;
 }
 
+static int get_reachable_list(struct object_array *src,
+      struct object_array *reachable)
+{
+ struct child_process cmd = CHILD_PROCESS_INIT;
+ int i, ret = do_reachable_revlist(&cmd, src, reachable);
+ struct object *o;
+ char namebuf[42]; /* ^ + SHA-1 + LF */
+
+ if (ret < 0)
+ return -1;
+
+ while ((i = read_in_full(cmd.out, namebuf, 41)) == 41) {
+ struct object_id sha1;
+
+ if (namebuf[40] != '\n' || get_oid_hex(namebuf, &sha1))
+ break;
+
+ o = lookup_object(sha1.hash);
+ if (o && o->type == OBJ_COMMIT) {
+ o->flags &= ~TMP_MARK;
+ }
+ }
+ for (i = get_max_object_index(); 0 < i; i--) {
+ o = get_indexed_object(i - 1);
+ if (o && o->type == OBJ_COMMIT &&
+    (o->flags & TMP_MARK)) {
+ add_object_array(o, NULL, reachable);
+ o->flags &= ~TMP_MARK;
+ }
+ }
+ close(cmd.out);
+
+ if (finish_command(&cmd))
+ return -1;
+
+ return 0;
+}
+
 static int check_unreachable(struct object_array *src)
 {
  struct child_process cmd = CHILD_PROCESS_INIT;
- int i, ret = do_reachable_revlist(&cmd, src);
+ int i, ret = do_reachable_revlist(&cmd, src, NULL);
  char buf[1];
 
  if (ret < 0)
--
2.8.0.rc0.210.gd302cd2

--
To unsubscribe from this list: send the line "unsubscribe git" in
the body of a message to [hidden email]
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Reply | Threaded
Open this post in threaded view
|

[PATCH 26/26] fetch, upload-pack: --deepen=N extends shallow boundary by N commits

Duy Nguyen
In reply to this post by Duy Nguyen
In git-fetch, --depth argument is always relative with the latest
remote refs. This makes it a bit difficult to cover this use case,
where the user wants to make the shallow history, say 3 levels
deeper. It would work if remote refs have not moved yet, but nobody
can guarantee that, especially when that use case is performed a
couple months after the last clone or "git fetch --depth". Also,
modifying shallow boundary using --depth does not work well with
clones created by --since or --not.

This patch fixes that. A new argument --deepen=<N> will add <N> more (*)
parent commits to the current history regardless of where remote refs
are.

Have/Want negotiation is still respected. So if remote refs move, the
server will send two chunks: one between "have" and "want" and another
to extend shallow history. In theory, the client could send no "want"s
in order to get the second chunk only. But the protocol does not allow
that. Either you send no want lines, which means ls-remote; or you
have to send at least one want line that carries deep-relative to the
server..

The main work was done by Dongcan Jiang. I fixed it up here and there.
And of course all the bugs belong to me.

(*) We could even support --deepen=<N> where <N> is negative. In that
case we can cut some history from the shallow clone. This operation
(and --depth=<shorter depth>) does not require interaction with remote
side (and more complicated to implement as a result).

Helped-by: Duy Nguyen <[hidden email]>
Helped-by: Eric Sunshine <[hidden email]>
Helped-by: Junio C Hamano <[hidden email]>
Signed-off-by: Dongcan Jiang <[hidden email]>
Signed-off-by: Nguyễn Thái Ngọc Duy <[hidden email]>
---
 Documentation/fetch-options.txt                   |  5 +++++
 Documentation/git-fetch-pack.txt                  |  5 +++++
 Documentation/gitremote-helpers.txt               |  4 ++++
 Documentation/technical/protocol-capabilities.txt |  7 ++++++
 builtin/fetch-pack.c                              |  4 ++++
 builtin/fetch.c                                   | 14 +++++++++++-
 fetch-pack.c                                      |  3 +++
 fetch-pack.h                                      |  1 +
 remote-curl.c                                     | 14 +++++++++++-
 t/t5500-fetch-pack.sh                             | 23 ++++++++++++++++++++
 t/t5539-fetch-http-shallow.sh                     | 26 +++++++++++++++++++++++
 transport-helper.c                                |  1 +
 transport.c                                       |  4 ++++
 transport.h                                       |  4 ++++
 upload-pack.c                                     | 21 +++++++++++++-----
 15 files changed, 129 insertions(+), 7 deletions(-)

diff --git a/Documentation/fetch-options.txt b/Documentation/fetch-options.txt
index 7aa1285..3b91f15 100644
--- a/Documentation/fetch-options.txt
+++ b/Documentation/fetch-options.txt
@@ -14,6 +14,11 @@
  linkgit:git-clone[1]), deepen or shorten the history to the specified
  number of commits. Tags for the deepened commits are not fetched.
 
+--deepen=<depth>::
+ Similar to --depth, except it specifies the number of commits
+ from the current shallow boundary instead of from the tip of
+ each remote branch history.
+
 --shallow-since=<date>::
  Deepen or shorten the history of a shallow repository to
  include all reachable commits after <date>.
diff --git a/Documentation/git-fetch-pack.txt b/Documentation/git-fetch-pack.txt
index 4d15b04..c20958f 100644
--- a/Documentation/git-fetch-pack.txt
+++ b/Documentation/git-fetch-pack.txt
@@ -96,6 +96,11 @@ be in a separate packet, and the list must end with a flush packet.
  exclude commits reachable from a specified remote branch or tag.
  This option can be specified multiple times.
 
+--deepen-relative::
+ Argument --depth specifies the number of commits from the
+ current shallow boundary instead of from the tip of each
+ remote branch history.
+
 --no-progress::
  Do not show the progress.
 
diff --git a/Documentation/gitremote-helpers.txt b/Documentation/gitremote-helpers.txt
index 75bb638..6fca268 100644
--- a/Documentation/gitremote-helpers.txt
+++ b/Documentation/gitremote-helpers.txt
@@ -422,6 +422,10 @@ set by Git if the remote helper has the 'option' capability.
  Deepens the history of a shallow repository excluding ref.
  Multiple options add up.
 
+'option deepen-relative {'true'|'false'}::
+ Deepens the history of a shallow repository relative to
+ current boundary. Only valid when used with "option depth".
+
 'option followtags' {'true'|'false'}::
  If enabled the helper should automatically fetch annotated
  tag objects if the object the tag points at was transferred
diff --git a/Documentation/technical/protocol-capabilities.txt b/Documentation/technical/protocol-capabilities.txt
index 0e6b57d..4fd6dcc 100644
--- a/Documentation/technical/protocol-capabilities.txt
+++ b/Documentation/technical/protocol-capabilities.txt
@@ -197,6 +197,13 @@ specific revision, instead of depth. Internally it's equivalent of
 doing "rev-list --not <rev>" on the server side. "deepen-not"
 cannot be used with "deepen", but can be used with "deepen-since".
 
+deepen-relative
+---------------
+
+If this capability is requested by the client, the semantics of
+"deepen" command is changed. The "depth" argument is the depth from
+the current shallow boundary, instead of the depth from remote refs.
+
 no-progress
 -----------
 
diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c
index 07570be..8265348 100644
--- a/builtin/fetch-pack.c
+++ b/builtin/fetch-pack.c
@@ -113,6 +113,10 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
  string_list_append(&deepen_not, arg);
  continue;
  }
+ if (!strcmp(arg, "--deepen-relative")) {
+ args.deepen_relative = 1;
+ continue;
+ }
  if (!strcmp("--no-progress", arg)) {
  args.no_progress = 1;
  continue;
diff --git a/builtin/fetch.c b/builtin/fetch.c
index 154b9ad..68b44ba 100644
--- a/builtin/fetch.c
+++ b/builtin/fetch.c
@@ -34,7 +34,7 @@ static int fetch_prune_config = -1; /* unspecified */
 static int prune = -1; /* unspecified */
 #define PRUNE_BY_DEFAULT 0 /* do we prune by default? */
 
-static int all, append, dry_run, force, keep, multiple, update_head_ok, verbosity;
+static int all, append, dry_run, force, keep, multiple, update_head_ok, verbosity, deepen_relative;
 static int progress = -1, recurse_submodules = RECURSE_SUBMODULES_DEFAULT;
 static int tags = TAGS_DEFAULT, unshallow, update_shallow, deepen;
 static int max_children = 1;
@@ -129,6 +129,8 @@ static struct option builtin_fetch_options[] = {
  { OPTION_CALLBACK, 0, "shallow-exclude", NULL, N_("revision"),
     N_("deepen history of shallow clone by excluding rev"),
     PARSE_OPT_NONEG, option_parse_deepen_not },
+ OPT_INTEGER(0, "deepen", &deepen_relative,
+    N_("deepen history of shallow clone")),
  { OPTION_SET_INT, 0, "unshallow", &unshallow, NULL,
    N_("convert to a complete repository"),
    PARSE_OPT_NONEG | PARSE_OPT_NOARG, NULL, 1 },
@@ -889,6 +891,8 @@ static struct transport *prepare_transport(struct remote *remote, int deepen)
  if (deepen && deepen_not.nr)
  set_option(transport, TRANS_OPT_DEEPEN_NOT,
    (const char *)&deepen_not);
+ if (deepen_relative)
+ set_option(transport, TRANS_OPT_DEEPEN_RELATIVE, "yes");
  if (update_shallow)
  set_option(transport, TRANS_OPT_UPDATE_SHALLOW, "yes");
  return transport;
@@ -914,6 +918,7 @@ static void backfill_tags(struct transport *transport, struct ref *ref_map)
 
  transport_set_option(transport, TRANS_OPT_FOLLOWTAGS, NULL);
  transport_set_option(transport, TRANS_OPT_DEPTH, "0");
+ transport_set_option(transport, TRANS_OPT_DEEPEN_RELATIVE, NULL);
  fetch_refs(transport, ref_map);
 
  if (gsecondary) {
@@ -1185,6 +1190,13 @@ int cmd_fetch(int argc, const char **argv, const char *prefix)
  argc = parse_options(argc, argv, prefix,
      builtin_fetch_options, builtin_fetch_usage, 0);
 
+ if (deepen_relative) {
+ if (deepen_relative < 0)
+ die(_("Negative depth in --deepen is not supported"));
+ if (depth)
+ die(_("--deepen and --depth are mutually exclusive"));
+ depth = xstrfmt("%d", deepen_relative);
+ }
  if (unshallow) {
  if (depth)
  die(_("--depth and --unshallow cannot be used together"));
diff --git a/fetch-pack.c b/fetch-pack.c
index ad7d00f..e2a235f 100644
--- a/fetch-pack.c
+++ b/fetch-pack.c
@@ -324,6 +324,7 @@ static int find_common(struct fetch_pack_args *args,
  if (no_done)            strbuf_addstr(&c, " no-done");
  if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
  if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
+ if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative");
  if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
  if (args->no_progress)   strbuf_addstr(&c, " no-progress");
  if (args->include_tag)   strbuf_addstr(&c, " include-tag");
@@ -883,6 +884,8 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args,
  deepen_not_ok = 1;
  else if (args->deepen_not)
  die(_("Server does not support --shallow-exclude"));
+ if (!server_supports("deepen-relative") && args->deepen_relative)
+ die(_("Server does not support --deepen"));
 
  if (everything_local(args, &ref, sought, nr_sought)) {
  packet_flush(fd[1]);
diff --git a/fetch-pack.h b/fetch-pack.h
index 144301f..c912e3d 100644
--- a/fetch-pack.h
+++ b/fetch-pack.h
@@ -12,6 +12,7 @@ struct fetch_pack_args {
  int depth;
  const char *deepen_since;
  const struct string_list *deepen_not;
+ unsigned deepen_relative:1;
  unsigned quiet:1;
  unsigned keep_pack:1;
  unsigned lock_pack:1;
diff --git a/remote-curl.c b/remote-curl.c
index 71122ac..3f1a8f5 100644
--- a/remote-curl.c
+++ b/remote-curl.c
@@ -30,7 +30,8 @@ struct options {
  dry_run : 1,
  thin : 1,
  /* One of the SEND_PACK_PUSH_CERT_* constants. */
- push_cert : 2;
+ push_cert : 2,
+ deepen_relative : 1;
 };
 static struct options options;
 static struct string_list cas_options = STRING_LIST_INIT_DUP;
@@ -70,6 +71,15 @@ static int set_option(const char *name, const char *value)
  string_list_append(&options.deepen_not, value);
  return 0;
  }
+ else if (!strcmp(name, "deepen-relative")) {
+ if (!strcmp(value, "true"))
+ options.deepen_relative = 1;
+ else if (!strcmp(value, "false"))
+ options.deepen_relative = 0;
+ else
+ return -1;
+ return 0;
+ }
  else if (!strcmp(name, "followtags")) {
  if (!strcmp(value, "true"))
  options.followtags = 1;
@@ -764,6 +774,8 @@ static int fetch_git(struct discovery *heads,
  for (i = 0; i < options.deepen_not.nr; i++)
  argv_array_pushf(&args, "--shallow-exclude=%s",
  options.deepen_not.items[i].string);
+ if (options.deepen_relative && options.depth)
+ argv_array_push(&args, "--deepen-relative");
  argv_array_push(&args, url.buf);
 
  for (i = 0; i < nr_heads; i++) {
diff --git a/t/t5500-fetch-pack.sh b/t/t5500-fetch-pack.sh
index a3fe5ca..f512098 100755
--- a/t/t5500-fetch-pack.sh
+++ b/t/t5500-fetch-pack.sh
@@ -683,4 +683,27 @@ test_expect_success 'fetch exclude tag one' '
  test_cmp expected actual
 '
 
+test_expect_success 'fetching deepen' '
+ test_create_repo shallow-deepen &&
+ (
+ cd shallow-deepen &&
+ test_commit one &&
+ test_commit two &&
+ test_commit three &&
+ git clone --depth 1 "file://$(pwd)/." deepen &&
+ test_commit four &&
+ git -C deepen log --pretty=tformat:%s master >actual &&
+ echo three >expected &&
+ test_cmp expected actual &&
+ git -C deepen fetch --deepen=1 &&
+ git -C deepen log --pretty=tformat:%s origin/master >actual &&
+ cat >expected <<-\EOF &&
+ four
+ three
+ two
+ EOF
+ test_cmp expected actual
+ )
+'
+
 test_done
diff --git a/t/t5539-fetch-http-shallow.sh b/t/t5539-fetch-http-shallow.sh
index f71573d..25f8968 100755
--- a/t/t5539-fetch-http-shallow.sh
+++ b/t/t5539-fetch-http-shallow.sh
@@ -121,5 +121,31 @@ test_expect_success 'fetch exclude tag one' '
  test_cmp expected actual
 '
 
+test_expect_success 'fetching deepen' '
+ test_create_repo shallow-deepen &&
+ (
+ cd shallow-deepen &&
+ test_commit one &&
+ test_commit two &&
+ test_commit three &&
+ mv .git "$HTTPD_DOCUMENT_ROOT_PATH/shallow-deepen.git" &&
+ git clone --depth 1 $HTTPD_URL/smart/shallow-deepen.git deepen &&
+ mv "$HTTPD_DOCUMENT_ROOT_PATH/shallow-deepen.git" .git &&
+ test_commit four &&
+ git -C deepen log --pretty=tformat:%s master >actual &&
+ echo three >expected &&
+ test_cmp expected actual &&
+ mv .git "$HTTPD_DOCUMENT_ROOT_PATH/shallow-deepen.git" &&
+ git -C deepen fetch --deepen=1 &&
+ git -C deepen log --pretty=tformat:%s origin/master >actual &&
+ cat >expected <<-\EOF &&
+ four
+ three
+ two
+ EOF
+ test_cmp expected actual
+ )
+'
+
 stop_httpd
 test_done
diff --git a/transport-helper.c b/transport-helper.c
index d644568..b894b60 100644
--- a/transport-helper.c
+++ b/transport-helper.c
@@ -258,6 +258,7 @@ static const char *boolean_options[] = {
  TRANS_OPT_THIN,
  TRANS_OPT_KEEP,
  TRANS_OPT_FOLLOWTAGS,
+ TRANS_OPT_DEEPEN_RELATIVE
  };
 
 static int strbuf_set_helper_option(struct helper_data *data,
diff --git a/transport.c b/transport.c
index 3e6f3aa..3e76a9a 100644
--- a/transport.c
+++ b/transport.c
@@ -157,6 +157,9 @@ static int set_git_option(struct git_transport_options *opts,
  } else if (!strcmp(name, TRANS_OPT_DEEPEN_NOT)) {
  opts->deepen_not = (const struct string_list *)value;
  return 0;
+ } else if (!strcmp(name, TRANS_OPT_DEEPEN_RELATIVE)) {
+ opts->deepen_relative = !!value;
+ return 0;
  }
  return 1;
 }
@@ -213,6 +216,7 @@ static int fetch_refs_via_pack(struct transport *transport,
  args.depth = data->options.depth;
  args.deepen_since = data->options.deepen_since;
  args.deepen_not = data->options.deepen_not;
+ args.deepen_relative = data->options.deepen_relative;
  args.check_self_contained_and_connected =
  data->options.check_self_contained_and_connected;
  args.cloning = transport->cloning;
diff --git a/transport.h b/transport.h
index ab61932..bdc3518 100644
--- a/transport.h
+++ b/transport.h
@@ -14,6 +14,7 @@ struct git_transport_options {
  unsigned check_self_contained_and_connected : 1;
  unsigned self_contained_and_connected : 1;
  unsigned update_shallow : 1;
+ unsigned deepen_relative : 1;
  int depth;
  const char *deepen_since;
  const struct string_list *deepen_not;
@@ -181,6 +182,9 @@ int transport_restrict_protocols(void);
 /* Limit the depth of the fetch based on revs if not null */
 #define TRANS_OPT_DEEPEN_NOT "deepen-not"
 
+/* Limit the deepen of the fetch if not null */
+#define TRANS_OPT_DEEPEN_RELATIVE "deepen-relative"
+
 /* Aggressively fetch annotated tags if possible */
 #define TRANS_OPT_FOLLOWTAGS "followtags"
 
diff --git a/upload-pack.c b/upload-pack.c
index 9bd590c..18b914a 100644
--- a/upload-pack.c
+++ b/upload-pack.c
@@ -32,6 +32,7 @@ static const char upload_pack_usage[] = "git upload-pack [--strict] [--timeout=<
 
 static unsigned long oldest_have;
 
+static int deepen_relative;
 static int multi_ack;
 static int no_done;
 static int use_thin_pack, use_ofs_delta, use_include_tag;
@@ -652,7 +653,8 @@ static void send_unshallow(const struct object_array *shallows)
  }
 }
 
-static void deepen(int depth, const struct object_array *shallows)
+static void deepen(int depth, int deepen_relative,
+   struct object_array *shallows)
 {
  struct commit_list *result = NULL;
  int i;
@@ -661,7 +663,14 @@ static void deepen(int depth, const struct object_array *shallows)
  struct object *object = shallows->objects[i].item;
  object->flags |= NOT_SHALLOW;
  }
- else
+ else if (deepen_relative) {
+ struct object_array reachable_shallows = OBJECT_ARRAY_INIT;
+ get_reachable_list(shallows, &reachable_shallows);
+ result = get_shallow_commits(&reachable_shallows,
+     depth + 1,
+     SHALLOW, NOT_SHALLOW);
+ object_array_clear(&reachable_shallows);
+ } else
  result = get_shallow_commits(&want_obj, depth,
      SHALLOW, NOT_SHALLOW);
  send_shallow(result);
@@ -753,6 +762,8 @@ static void receive_needs(void)
 
  features = arg + 40;
 
+ if (parse_feature_request(features, "deepen-relative"))
+ deepen_relative = 1;
  if (parse_feature_request(features, "multi_ack_detailed"))
  multi_ack = 2;
  else if (parse_feature_request(features, "multi_ack"))
@@ -802,7 +813,7 @@ static void receive_needs(void)
  if (depth > 0 && deepen_rev_list)
  die("git upload-pack: deepen and deepen-since (or deepen-not) cannot be used together");
  if (depth > 0)
- deepen(depth, &shallows);
+ deepen(depth, deepen_relative, &shallows);
  else if (deepen_rev_list) {
  struct argv_array av = ARGV_ARRAY_INIT;
  int i;
@@ -873,8 +884,8 @@ static int send_ref(const char *refname, const struct object_id *oid,
     int flag, void *cb_data)
 {
  static const char *capabilities = "multi_ack thin-pack side-band"
- " side-band-64k ofs-delta shallow deepen-since deepen-not no-progress"
- " include-tag multi_ack_detailed";
+ " side-band-64k ofs-delta shallow deepen-since deepen-not"
+ " deepen-relative no-progress include-tag multi_ack_detailed";
  const char *refname_nons = strip_namespace(refname);
  struct object_id peeled;
 
--
2.8.0.rc0.210.gd302cd2

--
To unsubscribe from this list: send the line "unsubscribe git" in
the body of a message to [hidden email]
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Reply | Threaded
Open this post in threaded view
|

Re: [PATCH 00/26] nd/shallow-deepen updates

Junio C Hamano
In reply to this post by Duy Nguyen
Thanks, picked up and re-queued.

--
To unsubscribe from this list: send the line "unsubscribe git" in
the body of a message to [hidden email]
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Reply | Threaded
Open this post in threaded view
|

Re: [PATCH 01/26] remote-curl.c: convert fetch_git() to use argv_array

Eric Sunshine
In reply to this post by Duy Nguyen
On Wed, Apr 13, 2016 at 8:54 AM, Nguyễn Thái Ngọc Duy <[hidden email]> wrote:

> Signed-off-by: Nguyễn Thái Ngọc Duy <[hidden email]>
> ---
> diff --git a/remote-curl.c b/remote-curl.c
> @@ -726,37 +726,30 @@ static int fetch_git(struct discovery *heads,
>         char *depth_arg = NULL;
> -       int argc = 0, i, err;
> -       const char *argv[17];
> +       int i, err;
> +       struct argv_array args = ARGV_ARRAY_INIT;
> [...]
>         if (options.verbosity >= 3) {
> -               argv[argc++] = "-v";
> -               argv[argc++] = "-v";
> +               argv_array_push(&args, "-v");
> +               argv_array_push(&args, "-v");

A bit more natural might have been:

    argv_array_pushl(&args, "-v", "-v", NULL);

though the diff becomes noisier when the braces get dropped. Not worth
a re-roll, of course.

>         }
> [...]
> -       if (options.depth) {
> -               struct strbuf buf = STRBUF_INIT;
> -               strbuf_addf(&buf, "--depth=%lu", options.depth);
> -               depth_arg = strbuf_detach(&buf, NULL);
> -               argv[argc++] = depth_arg;
> -       }
> -       argv[argc++] = url.buf;
> -       argv[argc++] = NULL;
> +               argv_array_push(&args, "--no-progress");
> +       if (options.depth)
> +               argv_array_pushf(&args, "--depth=%lu", options.depth);

'depth_arg' becomes unused with this change and can be retired...

> +       argv_array_push(&args, url.buf);
> @@ -779,6 +772,7 @@ static int fetch_git(struct discovery *heads,
>         strbuf_release(&rpc.result);
>         strbuf_release(&preamble);
>         free(depth_arg);

...along with this unnecessary free().

> +       argv_array_clear(&args);
>         return err;
>  }
--
To unsubscribe from this list: send the line "unsubscribe git" in
the body of a message to [hidden email]
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Reply | Threaded
Open this post in threaded view
|

Re: [PATCH 02/26] transport-helper.c: refactor set_helper_option()

Eric Sunshine
In reply to this post by Duy Nguyen
On Wed, Apr 13, 2016 at 8:54 AM, Nguyễn Thái Ngọc Duy <[hidden email]> wrote:

> For now we can handle two types, string and boolean, in
> set_helper_option(). Later on we'll add string_list support, which does
> not fit well. The new function strbuf_set_helper_option() can be reused
> for a separate function that handles string-list.
>
> Signed-off-by: Nguyễn Thái Ngọc Duy <[hidden email]>
> ---
> diff --git a/transport-helper.c b/transport-helper.c
> @@ -260,6 +260,28 @@ static const char *boolean_options[] = {
> +static int strbuf_set_helper_option(struct helper_data *data,
> +                                   struct strbuf *buf)
> +{
> +       int ret;
> +
> +       sendline(data, buf);
> +       if (recvline(data, buf))
> +               exit(128);
> +
> +       if (!strcmp(buf->buf, "ok"))
> +               ret = 0;
> +       else if (starts_with(buf->buf, "error")) {
> +               ret = -1;
> +       } else if (!strcmp(buf->buf, "unsupported"))

You could use this opportunity to drop the unnecessary braces. (True,
doing so makes it something other than pure code movement, but it's
minor and probably a worthy cleanup.)

> +               ret = 1;
> +       else {
> +               warning("%s unexpectedly said: '%s'", data->name, buf->buf);
> +               ret = 1;
> +       }
> +       return ret;
> +}
> +
>  static int set_helper_option(struct transport *transport,
>                           const char *name, const char *value)
>  {
> @@ -291,20 +313,7 @@ static int set_helper_option(struct transport *transport,
>                 quote_c_style(value, &buf, NULL, 0);
>         strbuf_addch(&buf, '\n');
>
> -       sendline(data, &buf);
> -       if (recvline(data, &buf))
> -               exit(128);
> -
> -       if (!strcmp(buf.buf, "ok"))
> -               ret = 0;
> -       else if (starts_with(buf.buf, "error")) {
> -               ret = -1;
> -       } else if (!strcmp(buf.buf, "unsupported"))
> -               ret = 1;
> -       else {
> -               warning("%s unexpectedly said: '%s'", data->name, buf.buf);
> -               ret = 1;
> -       }
> +       ret = strbuf_set_helper_option(data, &buf);
>         strbuf_release(&buf);
>         return ret;
>  }
--
To unsubscribe from this list: send the line "unsubscribe git" in
the body of a message to [hidden email]
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Reply | Threaded
Open this post in threaded view
|

Re: [PATCH 07/26] upload-pack: use skip_prefix() instead of starts_with()

Eric Sunshine
In reply to this post by Duy Nguyen
On Wed, Apr 13, 2016 at 8:54 AM, Nguyễn Thái Ngọc Duy <[hidden email]> wrote:

> upload-pack: use skip_prefix() instead of starts_with()
>
> Signed-off-by: Nguyễn Thái Ngọc Duy <[hidden email]>
> ---
> diff --git a/upload-pack.c b/upload-pack.c
> @@ -403,8 +405,8 @@ static int get_common_commits(void)
> -               if (starts_with(line, "have ")) {
> -                       switch (got_sha1(line+5, sha1)) {
> +               if (skip_prefix(line, "have ", &arg)) {
> +                       switch (got_sha1(arg, sha1)) {

There's one more instance of starts_with() in upload-pack.c:main()
which could be given the same treatment:

    if (starts_with(arg, "--timeout=")) {
        timeout = atoi(arg+10);
        daemon_mode = 1;
        continue;
    }

Was its omission an oversight or intentional since it's not related to
the others?
--
To unsubscribe from this list: send the line "unsubscribe git" in
the body of a message to [hidden email]
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Reply | Threaded
Open this post in threaded view
|

Re: [PATCH 09/26] upload-pack: move rev-list code out of check_non_tip()

Eric Sunshine
In reply to this post by Duy Nguyen
On Wed, Apr 13, 2016 at 8:54 AM, Nguyễn Thái Ngọc Duy <[hidden email]> wrote:

> Signed-off-by: Nguyễn Thái Ngọc Duy <[hidden email]>
> ---
> diff --git a/upload-pack.c b/upload-pack.c
> @@ -451,7 +451,7 @@ static int is_our_ref(struct object *o)
> -static void check_non_tip(void)
> +static int check_unreachable(struct object_array *src)
> @@ -461,14 +461,6 @@ static void check_non_tip(void)
> -       /*
> -        * In the normal in-process case without
> -        * uploadpack.allowReachableSHA1InWant,
> -        * non-tip requests can never happen.
> -        */
> -       if (!stateless_rpc && !(allow_unadvertised_object_request & ALLOW_REACHABLE_SHA1))
> -               goto error;
> -
> @@ -476,7 +468,7 @@ static void check_non_tip(void)
>         if (start_command(&cmd))
> -               goto error;
> +               return 0;
> @@ -495,16 +487,16 @@ static void check_non_tip(void)
>                 if (write_in_full(cmd.in, namebuf, 42) < 0)
> -                       goto error;
> +                       return 0;
> [...]
> +       for (i = 0; i < src->nr; i++) {
> +               o = src->objects[i].item;
>                 if (is_our_ref(o))
>                         continue;
>                 memcpy(namebuf, oid_to_hex(&o->oid), GIT_SHA1_HEXSZ);
>                 if (write_in_full(cmd.in, namebuf, 41) < 0)
> -                       goto error;
> +                       return 0;
>         }
>         close(cmd.in);

It's a little bit ugly how this close() becomes disconnected after the
refactoring. In the original code, due to the consistent application
of 'goto error', it's reasonably obvious that skipping the close() is
not harmful since the error case die()'s unconditionally (according to
the comment). However, after the refactoring, the function simply
returns without invoking close(), so there's a disconnect, and it's
not obvious without looking at the caller that the program will die().

Also, if this function later gets a new caller, is that caller going
to need intimate knowledge about this potential descriptor leak?

> @@ -516,7 +508,7 @@ static void check_non_tip(void)
>          */
>         i = read_in_full(cmd.out, namebuf, 1);
>         if (i)
> -               goto error;
> +               return 0;
>         close(cmd.out);

Same observation.

It might be clearer if you retained the 'error' label and used it to
ensure that the descriptors get closed:

    cmd.in = -1;
    cmd.out = -1;
    ...
    if (...)
        goto error;
    ...
    /* All the non-tip ... */
    return 1;

error:
    if (cmd.in >= 0)
        close(cmd.in);
    if (cmd.out >= 0)
        close(cmd.out);
    return 0;

>         /*
> @@ -525,15 +517,29 @@ static void check_non_tip(void)
>          * even when it showed no commit.
>          */
>         if (finish_command(&cmd))
> -               goto error;
> +               return 0;

Here too. Does 'cmd' need to be cleaned up if the function bails
before finish_command()?

>         /* All the non-tip ones are ancestors of what we advertised */
> -       return;
> +       return 1;
> +}
> +
> +static void check_non_tip(void)
> +{
> +       int i;
> +
> +       /*
> +        * In the normal in-process case without
> +        * uploadpack.allowReachableSHA1InWant,
> +        * non-tip requests can never happen.
> +        */
> +       if (!stateless_rpc && !(allow_unadvertised_object_request & ALLOW_REACHABLE_SHA1))
> +               ;               /* error */
> +       else if (check_unreachable(&want_obj))
> +               return;

With the loss of the 'error' label (below), this logic becomes a bit
more difficult to follow. I wonder if it would help to invert the
sense of the conditional...

    if (stateless_rpc || ...)
        if (check_unreachable(...))
            return;

Or, perhaps, retain the 'error' label:

    if (!stateless_rpc && ...)
        goto error:
    if (check_unreachable(...))
        return;

error:
    /* Pick one ... */

I think I might find the latter a bit clearer, but it's highly subjective.

> -error:
>         /* Pick one of them (we know there at least is one) */
>         for (i = 0; i < want_obj.nr; i++) {
> -               o = want_obj.objects[i].item;
> +               struct object *o = want_obj.objects[i].item;
>                 if (!is_our_ref(o))
>                         die("git upload-pack: not our ref %s",
>                             oid_to_hex(&o->oid));
--
To unsubscribe from this list: send the line "unsubscribe git" in
the body of a message to [hidden email]
More majordomo info at  http://vger.kernel.org/majordomo-info.html
12