Re: [RFC] Move static archive dependencies into ld

[Michael Matz <matz@suse.de>]

Heyho,

On Mon, 5 Jun 2023, Howard Chu wrote:

> What needs to happen to move this RFC forward? There seems to be at 
> least some support for the idea (attached)

I was fiddling with also adding it to gold, and the support for threads 
there confused me, so I put it on hold a bit.  I did hack on it February, 
and got it working, but couldn't really convince myself that I got the 
dealing with the work-queues right.  See below.

(Of the BFD part already send back then I'm convinced).

So, if someone could look over the gold patch and tell me if the workqueue 
handling is somewhat okayish, we could proceed.  (The "interesting" thing 
about this is that I need to add items to a work-queue while it is 
processed, _at the place right after it_, not merely scheduled to run 
"eventually")


Ciao,
Michael.
-------------
diff --git a/gold/archive.cc b/gold/archive.cc
index 3983995d09a..5a439cfa412 100644
--- a/gold/archive.cc
+++ b/gold/archive.cc
@@ -190,7 +190,7 @@ const char Archive::sym64name[7] = { '/', 'S', 'Y', 'M', '6', '4', '/' };
 
 Archive::Archive(const std::string& name, Input_file* input_file,
                  bool is_thin_archive, Dirsearch* dirpath, Task* task)
-  : Library_base(task), name_(name), input_file_(input_file), armap_(),
+  : Library_base(task), libdeps_(NULL), name_(name), input_file_(input_file), armap_(),
     armap_names_(), extended_names_(), armap_checked_(), seen_offsets_(),
     members_(), is_thin_archive_(is_thin_archive), included_member_(false),
     nested_archives_(), dirpath_(dirpath), num_members_(0),
@@ -1005,6 +1005,7 @@ Archive::include_member(Symbol_table* symtab, Layout* layout,
         {
           obj->layout(symtab, layout, sd);
           obj->add_symbols(symtab, sd, layout);
+	  this->maybe_handle_libdeps();
 	  this->included_member_ = true;
         }
       delete sd;
@@ -1039,6 +1040,7 @@ Archive::include_member(Symbol_table* symtab, Layout* layout,
   if (pluginobj != NULL)
     {
       pluginobj->add_symbols(symtab, NULL, layout);
+      this->maybe_handle_libdeps();
       this->included_member_ = true;
       return true;
     }
@@ -1059,10 +1061,33 @@ Archive::include_member(Symbol_table* symtab, Layout* layout,
     obj->add_symbols(symtab, &sd, layout);
   }
 
+  this->maybe_handle_libdeps();
   this->included_member_ = true;
   return true;
 }
 
+void
+Archive::maybe_handle_libdeps(void)
+{
+  if (this->included_member_)
+    return;
+  if (this->is_thin_archive_)
+    return;
+  fprintf(stderr, "handle_libdeps in %s\n", name_.c_str());
+  for (Archive::const_iterator p = this->begin();
+       p != this->end();
+       ++p)
+    {
+      fprintf(stderr, "  %u %u %u %s\n", (unsigned)p->off, (unsigned)p->nested_off, (unsigned)p->size, p->name.c_str());
+      if (p->name == "__.LIBDEP")
+	{
+	  const unsigned char* s = this->get_view(p->off + sizeof(Archive_header), p->size, false, false);
+	  fprintf(stderr, "    got it: %s\n", s);
+	  this->libdeps_ = reinterpret_cast<const char*>(s);
+	}
+    }
+}
+
 // Iterate over all unused symbols, and call the visitor class V for each.
 
 void
@@ -1119,6 +1144,87 @@ Add_archive_symbols::locks(Task_locker* tl)
   tl->add(this, this->archive_->token());
 }
 
+/* Turn a string into an argvec.  Copied from the original implementation
+   in the plugin, hence written in C.  */
+
+static char **
+str2vec (char *in)
+{
+  char **res;
+  char *s, *first, *end;
+  char *sq, *dq;
+  int i;
+
+  end = in + strlen (in);
+  s = in;
+  while (isspace ((unsigned char) *s)) s++;
+  first = s;
+
+  i = 1;
+  while ((s = strchr (s, ' ')))
+    {
+      s++;
+      i++;
+    }
+  res = (char **)malloc ((i+1) * sizeof (char *));
+  if (!res)
+    return res;
+
+  i = 0;
+  sq = NULL;
+  dq = NULL;
+  res[0] = first;
+  for (s = first; *s; s++)
+    {
+      if (*s == '\\')
+	{
+	  memmove (s, s+1, end-s-1);
+	  end--;
+	}
+      if (isspace ((unsigned char) *s))
+	{
+	  if (sq || dq)
+	    continue;
+	  *s++ = '\0';
+	  while (isspace ((unsigned char) *s)) s++;
+	  if (*s)
+	    res[++i] = s;
+	}
+      if (*s == '\'' && !dq)
+	{
+	  if (sq)
+	    {
+	      memmove (sq, sq+1, s-sq-1);
+	      memmove (s-2, s+1, end-s-1);
+	      end -= 2;
+	      s--;
+	      sq = NULL;
+	    }
+	  else
+	    {
+	      sq = s;
+	    }
+	}
+      if (*s == '"' && !sq)
+	{
+	  if (dq)
+	    {
+	      memmove (dq, dq+1, s-dq-1);
+	      memmove (s-2, s+1, end-s-1);
+	      end -= 2;
+	      s--;
+	      dq = NULL;
+	    }
+	  else
+	    {
+	      dq = s;
+	    }
+	}
+    }
+  res[++i] = NULL;
+  return res;
+}
+
 void
 Add_archive_symbols::run(Workqueue* workqueue)
 {
@@ -1135,6 +1241,79 @@ Add_archive_symbols::run(Workqueue* workqueue)
   bool added = this->archive_->add_symbols(this->symtab_, this->layout_,
 					   this->input_objects_,
 					   this->mapfile_);
+  if (added && this->archive_->libdeps_)
+    {
+      std::vector<Input_argument *> newinputs;
+      char **vec;
+      vec = str2vec (strdup (this->archive_->libdeps_));
+      if (vec)
+	{
+	  std::string extra_search_path = "";
+	  int i;
+	  for (i = 0; vec[i]; i++)
+	    {
+	      if (vec[i][0] != '-')
+		fprintf (stderr, "ignoring libdep argument %s", vec[i]);
+	      else if (vec[i][1] == 'l')
+		{
+		  Input_file_argument file(
+			vec[i]+2,
+			Input_file_argument::INPUT_FILE_TYPE_LIBRARY,
+			extra_search_path.c_str(),
+			false,
+			this->archive_->input_file()->options());
+		  Input_argument* input_argument = new Input_argument(file);
+		  newinputs.push_back(input_argument);
+		}
+	      else if (vec[i][1] == 'L')
+		extra_search_path = vec[i]+2;
+	      else
+		fprintf (stderr, "ignoring libdep argument %s", vec[i]);
+	    }
+	  free (vec);
+	}
+
+      Task_token* this_blocker = NULL;
+      for (std::vector<Input_argument*>::const_iterator i = newinputs.begin();
+	   i != newinputs.end();
+	   ++i)
+	{
+	  const Input_argument* arg = *i;
+
+	  Task_token* next_blocker;
+	  if (i != newinputs.end() - 1)
+	    {
+	      next_blocker = new Task_token(true);
+	      next_blocker->add_blocker();
+	      //fprintf(stderr, "XXX create new blocker %p\n", next_blocker);
+	    }
+	  else
+	    {
+	      next_blocker = this->next_blocker_;
+	      /* The originally next task is blocked on _this_ task (about
+		 to end right soon) and the last inserted new task.  */
+	      //next_blocker->add_blocker();
+	      workqueue->add_blocker(next_blocker);
+	      //fprintf(stderr, "XXX reuse old blocker %p\n", next_blocker);
+	    }
+
+	  //fprintf(stderr, "XXX queue with this_blocker %p\n", this_blocker);
+	  workqueue->queue_soon(new Read_symbols(this->input_objects_,
+						 this->symtab_,
+						 this->layout_,
+						 this->dirpath_,
+						 0,
+						 this->mapfile_,
+						 arg,
+						 NULL,
+						 NULL,
+						 this_blocker,
+						 next_blocker));
+	  this_blocker = next_blocker;
+	}
+    }
+
+
   this->archive_->unlock_nested_archives();
 
   this->archive_->release();
diff --git a/gold/archive.h b/gold/archive.h
index 6e20d9c5f14..488563e7624 100644
--- a/gold/archive.h
+++ b/gold/archive.h
@@ -264,6 +264,8 @@ class Archive : public Library_base
   no_export()
   { return this->no_export_; }
 
+  const char *libdeps_;
+
  private:
   Archive(const Archive&);
   Archive& operator=(const Archive&);
@@ -340,6 +342,10 @@ class Archive : public Library_base
   include_member(Symbol_table*, Layout*, Input_objects*, off_t off,
 		 Mapfile*, Symbol*, const char* why);
 
+  // Possibly search for and deal with recorded library dependencies.
+  void
+  maybe_handle_libdeps(void);
+
   // Return whether we found this archive by searching a directory.
   bool
   searched_for() const
diff --git a/gold/token.h b/gold/token.h
index b58fcc2e1cd..723729c84b8 100644
--- a/gold/token.h
+++ b/gold/token.h
@@ -97,6 +97,7 @@ class Task_token
 
   ~Task_token()
   {
+    //fprintf(stderr, "XXX dtor blocker %p\n", this);
     gold_assert(this->blockers_ == 0);
     gold_assert(this->writer_ == NULL);
   }
@@ -121,6 +122,7 @@ class Task_token
   void
   add_writer(const Task* t)
   {
+    //fprintf(stderr, "XXX add writer %p\n", this);
     gold_assert(!this->is_blocker_ && this->writer_ == NULL);
     this->writer_ = t;
   }
@@ -158,6 +160,7 @@ class Task_token
   bool
   remove_blocker()
   {
+    //fprintf(stderr, "XXX remove_blocker %p\n", this);
     gold_assert(this->is_blocker_ && this->blockers_ > 0);
     --this->blockers_;
     this->writer_ = NULL;
@@ -168,6 +171,7 @@ class Task_token
   bool
   is_blocked() const
   {
+    //fprintf(stderr, "XXX is_blocked %p\n", this);
     gold_assert(this->is_blocker_);
     return this->blockers_ > 0;
   }