From 05878158366e0c99e01e8abe5e91bce720ad5359 Mon Sep 17 00:00:00 2001
From: Tanna McClure <tmcclure@newrelic.com>
Date: Tue, 10 Oct 2023 13:52:18 -0700
Subject: [PATCH 01/18] Create async http instrumentation files

---
 .../agent/configuration/default_source.rb     |  8 +++++
 .../agent/instrumentation/async_http.rb       | 29 +++++++++++++++++++
 .../agent/instrumentation/async_http/chain.rb | 21 ++++++++++++++
 .../async_http/instrumentation.rb             | 12 ++++++++
 .../instrumentation/async_http/prepend.rb     | 13 +++++++++
 newrelic.yml                                  |  4 +++
 test/multiverse/suites/async_http/Envfile     |  9 ++++++
 .../async_http_instrumentation_test.rb        | 15 ++++++++++
 .../suites/async_http/config/newrelic.yml     | 19 ++++++++++++
 9 files changed, 130 insertions(+)
 create mode 100644 lib/new_relic/agent/instrumentation/async_http.rb
 create mode 100644 lib/new_relic/agent/instrumentation/async_http/chain.rb
 create mode 100644 lib/new_relic/agent/instrumentation/async_http/instrumentation.rb
 create mode 100644 lib/new_relic/agent/instrumentation/async_http/prepend.rb
 create mode 100644 test/multiverse/suites/async_http/Envfile
 create mode 100644 test/multiverse/suites/async_http/async_http_instrumentation_test.rb
 create mode 100644 test/multiverse/suites/async_http/config/newrelic.yml

diff --git a/lib/new_relic/agent/configuration/default_source.rb b/lib/new_relic/agent/configuration/default_source.rb
index 8203627c2b..c18b6be0ff 100644
--- a/lib/new_relic/agent/configuration/default_source.rb
+++ b/lib/new_relic/agent/configuration/default_source.rb
@@ -1370,6 +1370,14 @@ def self.enforce_fallback(allowed_values: nil, fallback: nil)
           :allowed_from_server => false,
           :description => 'Controls auto-instrumentation of `ActiveSupport::Logger` at start up. May be one of: `auto`, `prepend`, `chain`, `disabled`.'
         },
+        :'instrumentation.async_http' => {
+          :default => 'auto',
+          :public => true,
+          :type => String,
+          :dynamic_name => true,
+          :allowed_from_server => false,
+          :description => 'Controls auto-instrumentation of Async::HTTP at start up. May be one of: `auto`, `prepend`, `chain`, `disabled`.'
+        },
         :'instrumentation.bunny' => {
           :default => 'auto',
           :public => true,
diff --git a/lib/new_relic/agent/instrumentation/async_http.rb b/lib/new_relic/agent/instrumentation/async_http.rb
new file mode 100644
index 0000000000..2aa9e0334a
--- /dev/null
+++ b/lib/new_relic/agent/instrumentation/async_http.rb
@@ -0,0 +1,29 @@
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
+
+require_relative 'async_http/instrumentation'
+require_relative 'async_http/chain'
+require_relative 'async_http/prepend'
+
+DependencyDetection.defer do
+  named :'async_http'
+
+  depends_on do
+    # The class that needs to be defined to prepend/chain onto. This can be used
+    # to determine whether the library is installed.
+    defined?(Async::Http)
+    # Add any additional requirements to verify whether this instrumentation
+    # should be installed
+  end
+
+  executes do
+    NewRelic::Agent.logger.info('Installing async_http instrumentation')
+
+    if use_prepend?
+      prepend_instrument Async::Http, NewRelic::Agent::Instrumentation::AsyncHttp::Prepend
+    else
+      chain_instrument NewRelic::Agent::Instrumentation::AsyncHttp::Chain
+    end
+  end
+end
diff --git a/lib/new_relic/agent/instrumentation/async_http/chain.rb b/lib/new_relic/agent/instrumentation/async_http/chain.rb
new file mode 100644
index 0000000000..51d9b72800
--- /dev/null
+++ b/lib/new_relic/agent/instrumentation/async_http/chain.rb
@@ -0,0 +1,21 @@
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
+
+module NewRelic::Agent::Instrumentation
+  module AsyncHttp::Chain
+    def self.instrument!
+      ::Async::Http.class_eval do
+        include NewRelic::Agent::Instrumentation::AsyncHttp
+
+        alias_method(:method_to_instrument_without_new_relic, :method_to_instrument)
+
+        def method_to_instrument(*args)
+          method_to_instrument_with_new_relic(*args) do
+            method_to_instrument_without_new_relic(*args)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/new_relic/agent/instrumentation/async_http/instrumentation.rb b/lib/new_relic/agent/instrumentation/async_http/instrumentation.rb
new file mode 100644
index 0000000000..4a1f31fb66
--- /dev/null
+++ b/lib/new_relic/agent/instrumentation/async_http/instrumentation.rb
@@ -0,0 +1,12 @@
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
+
+module NewRelic::Agent::Instrumentation
+  module AsyncHttp
+    def method_to_instrument_with_new_relic(*args)
+      # add instrumentation content here
+      yield
+    end
+  end
+end
diff --git a/lib/new_relic/agent/instrumentation/async_http/prepend.rb b/lib/new_relic/agent/instrumentation/async_http/prepend.rb
new file mode 100644
index 0000000000..f024d8d1ad
--- /dev/null
+++ b/lib/new_relic/agent/instrumentation/async_http/prepend.rb
@@ -0,0 +1,13 @@
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
+
+module NewRelic::Agent::Instrumentation
+  module AsyncHttp::Prepend
+    include NewRelic::Agent::Instrumentation::AsyncHttp
+
+    def method_to_instrument(*args)
+      method_to_instrument_with_new_relic(*args) { super }
+    end
+  end
+end
diff --git a/newrelic.yml b/newrelic.yml
index 9d74fa56a6..61301b1490 100644
--- a/newrelic.yml
+++ b/newrelic.yml
@@ -382,6 +382,10 @@ common: &default_settings
   # prepend, chain, disabled.
   # instrumentation.bunny: auto
 
+# Controls auto-instrumentation of Async::HTTP at start up.
+# May be one of [auto|prepend|chain|disabled]
+# instrumentation.async_http: auto
+
   # Controls auto-instrumentation of the concurrent-ruby library at start up. May be
   # one of: auto, prepend, chain, disabled.
   # instrumentation.concurrent_ruby: auto
diff --git a/test/multiverse/suites/async_http/Envfile b/test/multiverse/suites/async_http/Envfile
new file mode 100644
index 0000000000..1add24a628
--- /dev/null
+++ b/test/multiverse/suites/async_http/Envfile
@@ -0,0 +1,9 @@
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
+
+instrumentation_methods :chain, :prepend
+
+gemfile <<~RB
+  gem 'async-http'
+RB
diff --git a/test/multiverse/suites/async_http/async_http_instrumentation_test.rb b/test/multiverse/suites/async_http/async_http_instrumentation_test.rb
new file mode 100644
index 0000000000..5e961c5fc9
--- /dev/null
+++ b/test/multiverse/suites/async_http/async_http_instrumentation_test.rb
@@ -0,0 +1,15 @@
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
+
+class AsyncHttpInstrumentationTest < Minitest::Test
+  def setup
+    @stats_engine = NewRelic::Agent.instance.stats_engine
+  end
+
+  def teardown
+    NewRelic::Agent.instance.stats_engine.clear_stats
+  end
+
+  # Add tests here
+end
diff --git a/test/multiverse/suites/async_http/config/newrelic.yml b/test/multiverse/suites/async_http/config/newrelic.yml
new file mode 100644
index 0000000000..51fa3fc536
--- /dev/null
+++ b/test/multiverse/suites/async_http/config/newrelic.yml
@@ -0,0 +1,19 @@
+---
+development:
+  error_collector:
+    enabled: true
+  apdex_t: 0.5
+  monitor_mode: true
+  license_key: bootstrap_newrelic_admin_license_key_000
+  instrumentation:
+    async_http: <%= $instrumentation_method %>
+  app_name: test
+  log_level: debug
+  host: 127.0.0.1
+  api_host: 127.0.0.1
+  transaction_trace:
+    record_sql: obfuscated
+    enabled: true
+    stack_trace_threshold: 0.5
+    transaction_threshold: 1.0
+  capture_params: false

From 37e3044017a318f227747fa2a3aabb59d13b3808 Mon Sep 17 00:00:00 2001
From: Tanna McClure <tmcclure@newrelic.com>
Date: Wed, 11 Oct 2023 12:55:33 -0700
Subject: [PATCH 02/18] add http wrapper for async http

---
 .../agent/http_clients/async_http_wrappers.rb | 66 +++++++++++++++++++
 1 file changed, 66 insertions(+)
 create mode 100644 lib/new_relic/agent/http_clients/async_http_wrappers.rb

diff --git a/lib/new_relic/agent/http_clients/async_http_wrappers.rb b/lib/new_relic/agent/http_clients/async_http_wrappers.rb
new file mode 100644
index 0000000000..76684c8e7d
--- /dev/null
+++ b/lib/new_relic/agent/http_clients/async_http_wrappers.rb
@@ -0,0 +1,66 @@
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
+
+require_relative 'abstract'
+require 'resolv'
+
+module NewRelic
+  module Agent
+    module HTTPClients
+      class AsyncHTTPResponse < AbstractResponse
+        def get_status_code
+          get_status_code_using(:status)
+        end
+      end
+
+      class AsyncHTTPRequest < AbstractRequest
+        def initialize(connection, method, url, headers)
+          @connection = connection
+          @method = method
+          @url = ::NewRelic::Agent::HTTPClients::URIUtil.parse_and_normalize_url(url)
+          @headers = headers
+        end
+
+        ASYNC_HTTP = 'Async::HTTP'
+        LHOST = 'host'
+        UHOST = 'Host'
+        COLON = ':'
+
+        def type
+          ASYNC_HTTP
+        end
+
+        def host_from_header
+          if hostname = (headers[LHOST] || headers[UHOST])
+            hostname.split(COLON).first
+          end
+        end
+
+        def host
+          host_from_header || uri.host.to_s
+        end
+
+        def [](key)
+          headers[key]
+        end
+
+        def []=(key, value)
+          headers[key] = value
+        end
+
+        def uri
+          @url
+        end
+
+        def headers
+          @headers
+        end
+
+        def method
+          @method
+        end
+      end
+    end
+  end
+end

From 221910089ac16e403c242b8c174ed705efe0a413 Mon Sep 17 00:00:00 2001
From: Tanna McClure <tmcclure@newrelic.com>
Date: Wed, 11 Oct 2023 12:59:07 -0700
Subject: [PATCH 03/18] instrument call method

---
 .../agent/instrumentation/async_http.rb       |  6 ++--
 .../agent/instrumentation/async_http/chain.rb | 12 ++++----
 .../async_http/instrumentation.rb             | 28 +++++++++++++++++--
 .../instrumentation/async_http/prepend.rb     |  6 ++--
 4 files changed, 40 insertions(+), 12 deletions(-)

diff --git a/lib/new_relic/agent/instrumentation/async_http.rb b/lib/new_relic/agent/instrumentation/async_http.rb
index 2aa9e0334a..0edd173abd 100644
--- a/lib/new_relic/agent/instrumentation/async_http.rb
+++ b/lib/new_relic/agent/instrumentation/async_http.rb
@@ -12,7 +12,7 @@
   depends_on do
     # The class that needs to be defined to prepend/chain onto. This can be used
     # to determine whether the library is installed.
-    defined?(Async::Http)
+    defined?(Async::HTTP)
     # Add any additional requirements to verify whether this instrumentation
     # should be installed
   end
@@ -20,8 +20,10 @@
   executes do
     NewRelic::Agent.logger.info('Installing async_http instrumentation')
 
+    require 'async/http/internet'
+    require 'new_relic/agent/http_clients/async_http_wrappers'
     if use_prepend?
-      prepend_instrument Async::Http, NewRelic::Agent::Instrumentation::AsyncHttp::Prepend
+      prepend_instrument Async::HTTP::Internet, NewRelic::Agent::Instrumentation::AsyncHttp::Prepend
     else
       chain_instrument NewRelic::Agent::Instrumentation::AsyncHttp::Chain
     end
diff --git a/lib/new_relic/agent/instrumentation/async_http/chain.rb b/lib/new_relic/agent/instrumentation/async_http/chain.rb
index 51d9b72800..3bfd956c20 100644
--- a/lib/new_relic/agent/instrumentation/async_http/chain.rb
+++ b/lib/new_relic/agent/instrumentation/async_http/chain.rb
@@ -2,17 +2,19 @@
 # See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
 # frozen_string_literal: true
 
+require_relative 'instrumentation'
+
 module NewRelic::Agent::Instrumentation
   module AsyncHttp::Chain
     def self.instrument!
-      ::Async::Http.class_eval do
+      ::Async::HTTP::Internet.class_eval do
         include NewRelic::Agent::Instrumentation::AsyncHttp
 
-        alias_method(:method_to_instrument_without_new_relic, :method_to_instrument)
+        alias_method(:call_without_new_relic, :call)
 
-        def method_to_instrument(*args)
-          method_to_instrument_with_new_relic(*args) do
-            method_to_instrument_without_new_relic(*args)
+        def call(*args)
+          call_with_new_relic(*args) do
+            call_without_new_relic(*args)
           end
         end
       end
diff --git a/lib/new_relic/agent/instrumentation/async_http/instrumentation.rb b/lib/new_relic/agent/instrumentation/async_http/instrumentation.rb
index 4a1f31fb66..fe05882c64 100644
--- a/lib/new_relic/agent/instrumentation/async_http/instrumentation.rb
+++ b/lib/new_relic/agent/instrumentation/async_http/instrumentation.rb
@@ -4,9 +4,31 @@
 
 module NewRelic::Agent::Instrumentation
   module AsyncHttp
-    def method_to_instrument_with_new_relic(*args)
-      # add instrumentation content here
-      yield
+    def call_with_new_relic(method, url, headers = nil, body = nil)
+      wrapped_request = NewRelic::Agent::HTTPClients::AsyncHTTPRequest.new(self, method, url, headers)
+
+      segment = NewRelic::Agent::Tracer.start_external_request_segment(
+        library: wrapped_request.type,
+        uri: wrapped_request.uri,
+        procedure: wrapped_request.method
+      )
+
+      begin
+        response = nil
+        segment.add_request_headers(wrapped_request)
+
+        NewRelic::Agent.disable_all_tracing do
+          response = NewRelic::Agent::Tracer.capture_segment_error(segment) do
+            yield
+          end
+        end
+
+        wrapped_response = NewRelic::Agent::HTTPClients::AsyncHTTPResponse.new(response)
+        segment.process_response_headers(wrapped_response)
+        response
+      ensure
+        segment&.finish
+      end
     end
   end
 end
diff --git a/lib/new_relic/agent/instrumentation/async_http/prepend.rb b/lib/new_relic/agent/instrumentation/async_http/prepend.rb
index f024d8d1ad..0291daaf8b 100644
--- a/lib/new_relic/agent/instrumentation/async_http/prepend.rb
+++ b/lib/new_relic/agent/instrumentation/async_http/prepend.rb
@@ -2,12 +2,14 @@
 # See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
 # frozen_string_literal: true
 
+require_relative 'instrumentation'
+
 module NewRelic::Agent::Instrumentation
   module AsyncHttp::Prepend
     include NewRelic::Agent::Instrumentation::AsyncHttp
 
-    def method_to_instrument(*args)
-      method_to_instrument_with_new_relic(*args) { super }
+    def call(*args)
+      call_with_new_relic(*args) { super }
     end
   end
 end

From 76915eb5f69aa877510cdfd07bd79cdedb5ffcf1 Mon Sep 17 00:00:00 2001
From: Tanna McClure <tmcclure@newrelic.com>
Date: Tue, 17 Oct 2023 15:43:46 -0700
Subject: [PATCH 04/18] http test cases working for async

---
 .../agent/http_clients/async_http_wrappers.rb |  8 ++
 .../agent/instrumentation/async_http/chain.rb |  6 +-
 .../async_http/instrumentation.rb             | 12 ++-
 .../instrumentation/async_http/prepend.rb     |  4 +-
 test/multiverse/suites/async_http/Envfile     |  1 +
 .../async_http_instrumentation_test.rb        | 87 +++++++++++++++++--
 6 files changed, 107 insertions(+), 11 deletions(-)

diff --git a/lib/new_relic/agent/http_clients/async_http_wrappers.rb b/lib/new_relic/agent/http_clients/async_http_wrappers.rb
index 76684c8e7d..bb5bb5b67d 100644
--- a/lib/new_relic/agent/http_clients/async_http_wrappers.rb
+++ b/lib/new_relic/agent/http_clients/async_http_wrappers.rb
@@ -12,6 +12,14 @@ class AsyncHTTPResponse < AbstractResponse
         def get_status_code
           get_status_code_using(:status)
         end
+
+        def [](key)
+          @wrapped_response.headers.to_h[key.downcase]&.first
+        end
+
+        def to_hash
+          @wrapped_response.headers.to_h
+        end
       end
 
       class AsyncHTTPRequest < AbstractRequest
diff --git a/lib/new_relic/agent/instrumentation/async_http/chain.rb b/lib/new_relic/agent/instrumentation/async_http/chain.rb
index 3bfd956c20..ad4109f042 100644
--- a/lib/new_relic/agent/instrumentation/async_http/chain.rb
+++ b/lib/new_relic/agent/instrumentation/async_http/chain.rb
@@ -12,9 +12,9 @@ def self.instrument!
 
         alias_method(:call_without_new_relic, :call)
 
-        def call(*args)
-          call_with_new_relic(*args) do
-            call_without_new_relic(*args)
+        def call(method, url, headers = nil, body = nil)
+          call_with_new_relic(method, url, headers, body) do |hdr|
+            call_without_new_relic(method, url, hdr, body)
           end
         end
       end
diff --git a/lib/new_relic/agent/instrumentation/async_http/instrumentation.rb b/lib/new_relic/agent/instrumentation/async_http/instrumentation.rb
index fe05882c64..f617278e20 100644
--- a/lib/new_relic/agent/instrumentation/async_http/instrumentation.rb
+++ b/lib/new_relic/agent/instrumentation/async_http/instrumentation.rb
@@ -4,7 +4,17 @@
 
 module NewRelic::Agent::Instrumentation
   module AsyncHttp
+
+    # from the async http doumentation:
+    # @parameter method [String] The request method, e.g. `GET`.
+    # @parameter url [String] The URL to request, e.g. `https://www.codeotaku.com`.
+    # @parameter headers [Hash | Protocol::HTTP::Headers] The headers to send with the request.
+    # @parameter body [String | Protocol::HTTP::Body] The body to send with the request.
+
+    # but their example has headers being an array??? weird, ig lets make sure we work ok with that
+    # like [[:content_type, "application/json"], [:accept, "application/json"]
     def call_with_new_relic(method, url, headers = nil, body = nil)
+      headers ||= {} # if it is nil, we need to make it a hash so we can insert headers
       wrapped_request = NewRelic::Agent::HTTPClients::AsyncHTTPRequest.new(self, method, url, headers)
 
       segment = NewRelic::Agent::Tracer.start_external_request_segment(
@@ -19,7 +29,7 @@ def call_with_new_relic(method, url, headers = nil, body = nil)
 
         NewRelic::Agent.disable_all_tracing do
           response = NewRelic::Agent::Tracer.capture_segment_error(segment) do
-            yield
+            yield(headers)
           end
         end
 
diff --git a/lib/new_relic/agent/instrumentation/async_http/prepend.rb b/lib/new_relic/agent/instrumentation/async_http/prepend.rb
index 0291daaf8b..b3bb3884ab 100644
--- a/lib/new_relic/agent/instrumentation/async_http/prepend.rb
+++ b/lib/new_relic/agent/instrumentation/async_http/prepend.rb
@@ -8,8 +8,8 @@ module NewRelic::Agent::Instrumentation
   module AsyncHttp::Prepend
     include NewRelic::Agent::Instrumentation::AsyncHttp
 
-    def call(*args)
-      call_with_new_relic(*args) { super }
+    def call(method, url, headers = nil, body = nil)
+      call_with_new_relic(method, url, headers, body) { |hdr| super(method, url, hdr, body) }
     end
   end
 end
diff --git a/test/multiverse/suites/async_http/Envfile b/test/multiverse/suites/async_http/Envfile
index 1add24a628..0c5778f6a4 100644
--- a/test/multiverse/suites/async_http/Envfile
+++ b/test/multiverse/suites/async_http/Envfile
@@ -6,4 +6,5 @@ instrumentation_methods :chain, :prepend
 
 gemfile <<~RB
   gem 'async-http'
+  gem 'rack'
 RB
diff --git a/test/multiverse/suites/async_http/async_http_instrumentation_test.rb b/test/multiverse/suites/async_http/async_http_instrumentation_test.rb
index 5e961c5fc9..fe93c9cd52 100644
--- a/test/multiverse/suites/async_http/async_http_instrumentation_test.rb
+++ b/test/multiverse/suites/async_http/async_http_instrumentation_test.rb
@@ -2,14 +2,91 @@
 # See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
 # frozen_string_literal: true
 
+require 'http_client_test_cases'
+
 class AsyncHttpInstrumentationTest < Minitest::Test
-  def setup
-    @stats_engine = NewRelic::Agent.instance.stats_engine
+  include HttpClientTestCases
+
+  def client_name
+    'Async::HTTP'
+  end
+
+  def timeout_error_class
+    Async::TimeoutError
+  end
+
+  def simulate_error_response
+    Async::HTTP::Client.any_instance.stubs(:call).raises(timeout_error_class.new('read timeout reached'))
+    get_response
+  end
+
+  def get_response(url = nil, headers = nil)
+    request_and_wait(:get, url || default_url, headers)
+  end
+
+  def request_and_wait(method, url, headers = nil, body = nil)
+    resp = nil
+    Async do
+      internet = Async::HTTP::Internet.new
+      resp = internet.send(method, url, headers)
+      @read_resp = resp&.read
+    rescue => e
+      puts "**************************ERROR: #{e}"
+    ensure
+      internet&.close
+    end
+    resp
+  end
+
+  def get_wrapped_response(url)
+    NewRelic::Agent::HTTPClients::AsyncHTTPResponse.new(get_response(url))
+  end
+
+  def head_response
+    request_and_wait(:head, default_url)
+  end
+
+  def post_response
+    request_and_wait(:post, default_url, nil, '')
+  end
+
+  def put_response
+    request_and_wait(:put, default_url, nil, '')
+  end
+
+  def delete_response
+    request_and_wait(:delete, default_url, nil, '')
+  end
+
+  def request_instance
+    NewRelic::Agent::HTTPClients::AsyncHTTPRequest.new(Async::HTTP::Internet.new, 'GET', default_url, {})
   end
 
-  def teardown
-    NewRelic::Agent.instance.stats_engine.clear_stats
+  def response_instance(headers = {})
+    resp = get_response(default_url, headers)
+    headers.each do |k, v|
+      resp.headers[k] = v
+    end
+
+    NewRelic::Agent::HTTPClients::AsyncHTTPResponse.new(resp)
   end
 
-  # Add tests here
+  def body(res)
+    @read_resp
+  end
+
+  def test_noticed_error_at_segment_and_txn_on_error
+    # skip
+    # Async gem does not allow the errors to escape the async block
+    # so the errors will never end up on the transaction, only ever the async http segment
+  end
+
+
+  # Test if headers are 
+  # nil
+  # array [ [key, value], [key, value] ]
+  # hash { key => value, key => value }
+  # http protocol header object
+
+
 end

From d24d89a4906cef783d74a6426da7677d43442f3f Mon Sep 17 00:00:00 2001
From: Tanna McClure <tmcclure@newrelic.com>
Date: Wed, 18 Oct 2023 13:33:52 -0700
Subject: [PATCH 05/18] ensure headers work if hash, array or protocol http
 object

---
 .../agent/http_clients/async_http_wrappers.rb | 15 ++++--
 .../async_http/instrumentation.rb             |  1 -
 .../async_http_instrumentation_test.rb        | 51 ++++++++++++++++---
 3 files changed, 57 insertions(+), 10 deletions(-)

diff --git a/lib/new_relic/agent/http_clients/async_http_wrappers.rb b/lib/new_relic/agent/http_clients/async_http_wrappers.rb
index bb5bb5b67d..fb36b55d66 100644
--- a/lib/new_relic/agent/http_clients/async_http_wrappers.rb
+++ b/lib/new_relic/agent/http_clients/async_http_wrappers.rb
@@ -40,7 +40,7 @@ def type
         end
 
         def host_from_header
-          if hostname = (headers[LHOST] || headers[UHOST])
+          if hostname = (self[LHOST] || self[UHOST])
             hostname.split(COLON).first
           end
         end
@@ -50,11 +50,20 @@ def host
         end
 
         def [](key)
-          headers[key]
+          return headers[key] unless headers.is_a?(Array)
+
+          headers.each do |header|
+            return header[1] if header[0].casecmp?(key)
+          end
+          nil
         end
 
         def []=(key, value)
-          headers[key] = value
+          if headers.is_a?(Array)
+            headers << [key, value]
+          else
+            headers[key] = value
+          end
         end
 
         def uri
diff --git a/lib/new_relic/agent/instrumentation/async_http/instrumentation.rb b/lib/new_relic/agent/instrumentation/async_http/instrumentation.rb
index f617278e20..b4aa2c6fc2 100644
--- a/lib/new_relic/agent/instrumentation/async_http/instrumentation.rb
+++ b/lib/new_relic/agent/instrumentation/async_http/instrumentation.rb
@@ -4,7 +4,6 @@
 
 module NewRelic::Agent::Instrumentation
   module AsyncHttp
-
     # from the async http doumentation:
     # @parameter method [String] The request method, e.g. `GET`.
     # @parameter url [String] The URL to request, e.g. `https://www.codeotaku.com`.
diff --git a/test/multiverse/suites/async_http/async_http_instrumentation_test.rb b/test/multiverse/suites/async_http/async_http_instrumentation_test.rb
index fe93c9cd52..00f21b6c54 100644
--- a/test/multiverse/suites/async_http/async_http_instrumentation_test.rb
+++ b/test/multiverse/suites/async_http/async_http_instrumentation_test.rb
@@ -76,17 +76,56 @@ def body(res)
   end
 
   def test_noticed_error_at_segment_and_txn_on_error
-    # skip
+    # skipping this test
     # Async gem does not allow the errors to escape the async block
     # so the errors will never end up on the transaction, only ever the async http segment
   end
 
+  def test_raw_synthetics_header_is_passed_along_if_present_array
+    with_config(:"cross_application_tracer.enabled" => true) do
+      in_transaction do
+        NewRelic::Agent::Tracer.current_transaction.raw_synthetics_header = 'boo'
 
-  # Test if headers are 
-  # nil
-  # array [ [key, value], [key, value] ]
-  # hash { key => value, key => value }
-  # http protocol header object
+        get_response(default_url, [%w[itsaheader itsavalue]])
 
+        assert_equal 'boo', server.requests.last['HTTP_X_NEWRELIC_SYNTHETICS']
+      end
+    end
+  end
+
+  def test_raw_synthetics_header_is_passed_along_if_present_hash
+    with_config(:"cross_application_tracer.enabled" => true) do
+      in_transaction do
+        NewRelic::Agent::Tracer.current_transaction.raw_synthetics_header = 'boo'
+
+        get_response(default_url, {'itsaheader' => 'itsavalue'})
+
+        assert_equal 'boo', server.requests.last['HTTP_X_NEWRELIC_SYNTHETICS']
+      end
+    end
+  end
+
+  def test_raw_synthetics_header_is_passed_along_if_present_protocol_header_hash
+    with_config(:"cross_application_tracer.enabled" => true) do
+      in_transaction do
+        NewRelic::Agent::Tracer.current_transaction.raw_synthetics_header = 'boo'
 
+        get_response(default_url, ::Protocol::HTTP::Headers[{'itsaheader' => 'itsavalue'}])
+
+        assert_equal 'boo', server.requests.last['HTTP_X_NEWRELIC_SYNTHETICS']
+      end
+    end
+  end
+
+  def test_raw_synthetics_header_is_passed_along_if_present_protocol_header_array
+    with_config(:"cross_application_tracer.enabled" => true) do
+      in_transaction do
+        NewRelic::Agent::Tracer.current_transaction.raw_synthetics_header = 'boo'
+
+        get_response(default_url, ::Protocol::HTTP::Headers[%w[itsaheader itsavalue]])
+
+        assert_equal 'boo', server.requests.last['HTTP_X_NEWRELIC_SYNTHETICS']
+      end
+    end
+  end
 end

From 8d0b0cbb014f8a24dbbcb59db9e72b74df4fcd2a Mon Sep 17 00:00:00 2001
From: Tanna McClure <tmcclure@newrelic.com>
Date: Wed, 18 Oct 2023 13:44:01 -0700
Subject: [PATCH 06/18] delete comments

---
 .../agent/instrumentation/async_http/instrumentation.rb   | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/lib/new_relic/agent/instrumentation/async_http/instrumentation.rb b/lib/new_relic/agent/instrumentation/async_http/instrumentation.rb
index b4aa2c6fc2..4049253467 100644
--- a/lib/new_relic/agent/instrumentation/async_http/instrumentation.rb
+++ b/lib/new_relic/agent/instrumentation/async_http/instrumentation.rb
@@ -4,14 +4,6 @@
 
 module NewRelic::Agent::Instrumentation
   module AsyncHttp
-    # from the async http doumentation:
-    # @parameter method [String] The request method, e.g. `GET`.
-    # @parameter url [String] The URL to request, e.g. `https://www.codeotaku.com`.
-    # @parameter headers [Hash | Protocol::HTTP::Headers] The headers to send with the request.
-    # @parameter body [String | Protocol::HTTP::Body] The body to send with the request.
-
-    # but their example has headers being an array??? weird, ig lets make sure we work ok with that
-    # like [[:content_type, "application/json"], [:accept, "application/json"]
     def call_with_new_relic(method, url, headers = nil, body = nil)
       headers ||= {} # if it is nil, we need to make it a hash so we can insert headers
       wrapped_request = NewRelic::Agent::HTTPClients::AsyncHTTPRequest.new(self, method, url, headers)

From ca6da2791a4aa94a73b91fd00ab2c027bf963514 Mon Sep 17 00:00:00 2001
From: Tanna McClure <tmcclure@newrelic.com>
Date: Wed, 18 Oct 2023 14:01:30 -0700
Subject: [PATCH 07/18] add changelog entry

---
 CHANGELOG.md | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 47b5cd1b3b..d6bb0799c4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,7 +2,12 @@
 
 ## dev
 
-Version <dev> gleans Docker container IDs from cgroups v2-based containers, records additional synthetics attributes, fixes an issue with Rails 7.1 that could cause duplicate log records to be sent to New Relic, and fixes a deprecation warning for the Sidekiq error handler.
+Version <dev> adds instrumentation for Async::HTTP, gleans Docker container IDs from cgroups v2-based containers, records additional synthetics attributes, fixes an issue with Rails 7.1 that could cause duplicate log records to be sent to New Relic, and fixes a deprecation warning for the Sidekiq error handler.
+
+- **Feature: Add instrumentation for Async::HTTP**
+
+  The agent will now record spans for Async::HTTP requests. [PR#2272](https://github.com/newrelic/newrelic-ruby-agent/pull/2272)
+
 
 - **Feature: Prevent the agent from starting in rails commands in Rails 7**
 

From 0b3edb6b65b7d18aede8cd6139c7e6754176205c Mon Sep 17 00:00:00 2001
From: Tanna McClure <tmcclure@newrelic.com>
Date: Thu, 19 Oct 2023 10:05:42 -0700
Subject: [PATCH 08/18] remove rescue from test

---
 .../suites/async_http/async_http_instrumentation_test.rb        | 2 --
 1 file changed, 2 deletions(-)

diff --git a/test/multiverse/suites/async_http/async_http_instrumentation_test.rb b/test/multiverse/suites/async_http/async_http_instrumentation_test.rb
index 00f21b6c54..2f81a14526 100644
--- a/test/multiverse/suites/async_http/async_http_instrumentation_test.rb
+++ b/test/multiverse/suites/async_http/async_http_instrumentation_test.rb
@@ -30,8 +30,6 @@ def request_and_wait(method, url, headers = nil, body = nil)
       internet = Async::HTTP::Internet.new
       resp = internet.send(method, url, headers)
       @read_resp = resp&.read
-    rescue => e
-      puts "**************************ERROR: #{e}"
     ensure
       internet&.close
     end

From 00f0fce2d0fe7f3105f743307c97a9f7d4a4b16f Mon Sep 17 00:00:00 2001
From: Tanna McClure <tmcclure@newrelic.com>
Date: Thu, 19 Oct 2023 10:16:51 -0700
Subject: [PATCH 09/18] Update newrelic.yml

Co-authored-by: James Bunch <fallwith@gmail.com>
---
 newrelic.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/newrelic.yml b/newrelic.yml
index 1d8db3fd2a..095419ac69 100644
--- a/newrelic.yml
+++ b/newrelic.yml
@@ -382,9 +382,9 @@ common: &default_settings
   # prepend, chain, disabled.
   # instrumentation.bunny: auto
 
-# Controls auto-instrumentation of Async::HTTP at start up.
-# May be one of [auto|prepend|chain|disabled]
-# instrumentation.async_http: auto
+  # Controls auto-instrumentation of Async::HTTP at start up.
+  # May be one of [auto|prepend|chain|disabled]
+  # instrumentation.async_http: auto
 
   # Controls auto-instrumentation of the concurrent-ruby library at start up. May be
   # one of: auto, prepend, chain, disabled.

From 33333f6ab2856675cfeacb20edcde6113ec361c3 Mon Sep 17 00:00:00 2001
From: Tanna McClure <tmcclure@newrelic.com>
Date: Thu, 19 Oct 2023 10:17:27 -0700
Subject: [PATCH 10/18] Update
 lib/new_relic/agent/http_clients/async_http_wrappers.rb

Co-authored-by: James Bunch <fallwith@gmail.com>
---
 lib/new_relic/agent/http_clients/async_http_wrappers.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/new_relic/agent/http_clients/async_http_wrappers.rb b/lib/new_relic/agent/http_clients/async_http_wrappers.rb
index fb36b55d66..394d8b7f48 100644
--- a/lib/new_relic/agent/http_clients/async_http_wrappers.rb
+++ b/lib/new_relic/agent/http_clients/async_http_wrappers.rb
@@ -14,7 +14,7 @@ def get_status_code
         end
 
         def [](key)
-          @wrapped_response.headers.to_h[key.downcase]&.first
+          to_hash[key.downcase]&.first
         end
 
         def to_hash

From 6be1dc13e329794a30251cdeb49c78095131555c Mon Sep 17 00:00:00 2001
From: Tanna McClure <tmcclure@newrelic.com>
Date: Thu, 19 Oct 2023 10:17:46 -0700
Subject: [PATCH 11/18] Update
 lib/new_relic/agent/instrumentation/async_http.rb

Co-authored-by: James Bunch <fallwith@gmail.com>
---
 lib/new_relic/agent/instrumentation/async_http.rb | 2 --
 1 file changed, 2 deletions(-)

diff --git a/lib/new_relic/agent/instrumentation/async_http.rb b/lib/new_relic/agent/instrumentation/async_http.rb
index 0edd173abd..483051f64a 100644
--- a/lib/new_relic/agent/instrumentation/async_http.rb
+++ b/lib/new_relic/agent/instrumentation/async_http.rb
@@ -13,8 +13,6 @@
     # The class that needs to be defined to prepend/chain onto. This can be used
     # to determine whether the library is installed.
     defined?(Async::HTTP)
-    # Add any additional requirements to verify whether this instrumentation
-    # should be installed
   end
 
   executes do

From fef6ba793936f2d306009c643c5ebdc316704704 Mon Sep 17 00:00:00 2001
From: Tanna McClure <tmcclure@newrelic.com>
Date: Thu, 19 Oct 2023 10:21:01 -0700
Subject: [PATCH 12/18] move require

---
 lib/new_relic/agent/instrumentation/async_http.rb               | 1 -
 .../agent/instrumentation/async_http/instrumentation.rb         | 2 ++
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/new_relic/agent/instrumentation/async_http.rb b/lib/new_relic/agent/instrumentation/async_http.rb
index 0edd173abd..850ede9bbe 100644
--- a/lib/new_relic/agent/instrumentation/async_http.rb
+++ b/lib/new_relic/agent/instrumentation/async_http.rb
@@ -21,7 +21,6 @@
     NewRelic::Agent.logger.info('Installing async_http instrumentation')
 
     require 'async/http/internet'
-    require 'new_relic/agent/http_clients/async_http_wrappers'
     if use_prepend?
       prepend_instrument Async::HTTP::Internet, NewRelic::Agent::Instrumentation::AsyncHttp::Prepend
     else
diff --git a/lib/new_relic/agent/instrumentation/async_http/instrumentation.rb b/lib/new_relic/agent/instrumentation/async_http/instrumentation.rb
index 4049253467..ceab6de327 100644
--- a/lib/new_relic/agent/instrumentation/async_http/instrumentation.rb
+++ b/lib/new_relic/agent/instrumentation/async_http/instrumentation.rb
@@ -2,6 +2,8 @@
 # See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
 # frozen_string_literal: true
 
+require 'new_relic/agent/http_clients/async_http_wrappers'
+
 module NewRelic::Agent::Instrumentation
   module AsyncHttp
     def call_with_new_relic(method, url, headers = nil, body = nil)

From 1906c1015fac8f5d3f1ce777a773c1fac075d188 Mon Sep 17 00:00:00 2001
From: Tanna McClure <tmcclure@newrelic.com>
Date: Thu, 19 Oct 2023 10:22:32 -0700
Subject: [PATCH 13/18] Update test/multiverse/suites/async_http/Envfile

Co-authored-by: James Bunch <fallwith@gmail.com>
---
 test/multiverse/suites/async_http/Envfile | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/test/multiverse/suites/async_http/Envfile b/test/multiverse/suites/async_http/Envfile
index 0c5778f6a4..6d4ac220a0 100644
--- a/test/multiverse/suites/async_http/Envfile
+++ b/test/multiverse/suites/async_http/Envfile
@@ -4,7 +4,16 @@
 
 instrumentation_methods :chain, :prepend
 
-gemfile <<~RB
-  gem 'async-http'
-  gem 'rack'
-RB
+ASYNC_HTTP_VERSIONS = [
+  nil,
+  '0.53.1'
+]
+
+def gem_list(async_http_version = nil)
+  <<~GEM_LIST
+    gem 'async-http'#{async_http_version}
+    gem 'rack'
+  GEM_LIST
+end
+
+create_gemfiles(ASYNC_HTTP_VERSIONS)

From f148bfd798679e73f653537009466323a49ac4db Mon Sep 17 00:00:00 2001
From: Tanna McClure <tmcclure@newrelic.com>
Date: Thu, 19 Oct 2023 10:25:31 -0700
Subject: [PATCH 14/18] add begin fro ruby 2.4

---
 .../async_http/async_http_instrumentation_test.rb    | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/test/multiverse/suites/async_http/async_http_instrumentation_test.rb b/test/multiverse/suites/async_http/async_http_instrumentation_test.rb
index 2f81a14526..3ea376388f 100644
--- a/test/multiverse/suites/async_http/async_http_instrumentation_test.rb
+++ b/test/multiverse/suites/async_http/async_http_instrumentation_test.rb
@@ -27,11 +27,13 @@ def get_response(url = nil, headers = nil)
   def request_and_wait(method, url, headers = nil, body = nil)
     resp = nil
     Async do
-      internet = Async::HTTP::Internet.new
-      resp = internet.send(method, url, headers)
-      @read_resp = resp&.read
-    ensure
-      internet&.close
+      begin
+        internet = Async::HTTP::Internet.new
+        resp = internet.send(method, url, headers)
+        @read_resp = resp&.read
+      ensure
+        internet&.close
+      end
     end
     resp
   end

From bad57ad0660f6cb40564195f99984c85ef0c3e56 Mon Sep 17 00:00:00 2001
From: Tanna McClure <tmcclure@newrelic.com>
Date: Thu, 19 Oct 2023 10:37:19 -0700
Subject: [PATCH 15/18] add to http client multiverse group

---
 test/multiverse/lib/multiverse/runner.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/multiverse/lib/multiverse/runner.rb b/test/multiverse/lib/multiverse/runner.rb
index 0ba27c3590..44e98313f5 100644
--- a/test/multiverse/lib/multiverse/runner.rb
+++ b/test/multiverse/lib/multiverse/runner.rb
@@ -104,7 +104,7 @@ def execute_suites(filter, opts)
       'database' => %w[elasticsearch mongo redis sequel],
       'rails' => %w[active_record active_record_pg active_support_broadcast_logger active_support_logger rails rails_prepend activemerchant],
       'frameworks' => %w[grape padrino roda sinatra],
-      'httpclients' => %w[curb excon httpclient],
+      'httpclients' => %w[async_http curb excon httpclient],
       'httpclients_2' => %w[typhoeus net_http httprb],
       'infinite_tracing' => ['infinite_tracing'],
 

From 3285c5bfae73159aa933f3bc126672bfbfc834ad Mon Sep 17 00:00:00 2001
From: Tanna McClure <tmcclure@newrelic.com>
Date: Thu, 19 Oct 2023 12:10:49 -0700
Subject: [PATCH 16/18] min version supported 0.59.0

---
 CHANGELOG.md                                      | 2 +-
 lib/new_relic/agent/instrumentation/async_http.rb | 4 +---
 test/multiverse/suites/async_http/Envfile         | 2 +-
 3 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index d6bb0799c4..e850f2f9e5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,7 +6,7 @@ Version <dev> adds instrumentation for Async::HTTP, gleans Docker container IDs
 
 - **Feature: Add instrumentation for Async::HTTP**
 
-  The agent will now record spans for Async::HTTP requests. [PR#2272](https://github.com/newrelic/newrelic-ruby-agent/pull/2272)
+  The agent will now record spans for Async::HTTP requests. Versions 0.59.0 and above of the async-http gem are supported. [PR#2272](https://github.com/newrelic/newrelic-ruby-agent/pull/2272)
 
 
 - **Feature: Prevent the agent from starting in rails commands in Rails 7**
diff --git a/lib/new_relic/agent/instrumentation/async_http.rb b/lib/new_relic/agent/instrumentation/async_http.rb
index d63b8ea987..be9c7e3046 100644
--- a/lib/new_relic/agent/instrumentation/async_http.rb
+++ b/lib/new_relic/agent/instrumentation/async_http.rb
@@ -10,9 +10,7 @@
   named :'async_http'
 
   depends_on do
-    # The class that needs to be defined to prepend/chain onto. This can be used
-    # to determine whether the library is installed.
-    defined?(Async::HTTP)
+    defined?(Async::HTTP) && Gem::Version.new(Async::HTTP::VERSION) >= Gem::Version.new('0.59.0')
   end
 
   executes do
diff --git a/test/multiverse/suites/async_http/Envfile b/test/multiverse/suites/async_http/Envfile
index 6d4ac220a0..ec92f43e28 100644
--- a/test/multiverse/suites/async_http/Envfile
+++ b/test/multiverse/suites/async_http/Envfile
@@ -6,7 +6,7 @@ instrumentation_methods :chain, :prepend
 
 ASYNC_HTTP_VERSIONS = [
   nil,
-  '0.53.1'
+  '0.59.0'
 ]
 
 def gem_list(async_http_version = nil)

From e911c78feb1ffc87524bc380577b133ab37820d7 Mon Sep 17 00:00:00 2001
From: Tanna McClure <tmcclure@newrelic.com>
Date: Thu, 19 Oct 2023 12:49:32 -0700
Subject: [PATCH 17/18] min ruby version 2.5

---
 test/multiverse/suites/async_http/Envfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/multiverse/suites/async_http/Envfile b/test/multiverse/suites/async_http/Envfile
index ec92f43e28..4488de786d 100644
--- a/test/multiverse/suites/async_http/Envfile
+++ b/test/multiverse/suites/async_http/Envfile
@@ -5,8 +5,8 @@
 instrumentation_methods :chain, :prepend
 
 ASYNC_HTTP_VERSIONS = [
-  nil,
-  '0.59.0'
+  [nil, 2.5],
+  ['0.59.0', 2.5]
 ]
 
 def gem_list(async_http_version = nil)

From 57c81fe4128dd8b3aa049adbb454b928513ea719 Mon Sep 17 00:00:00 2001
From: Tanna McClure <tmcclure@newrelic.com>
Date: Thu, 19 Oct 2023 17:13:20 -0700
Subject: [PATCH 18/18] Update
 lib/new_relic/agent/instrumentation/async_http.rb

Co-authored-by: James Bunch <fallwith@gmail.com>
---
 lib/new_relic/agent/instrumentation/async_http.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/new_relic/agent/instrumentation/async_http.rb b/lib/new_relic/agent/instrumentation/async_http.rb
index be9c7e3046..9192ac5ca6 100644
--- a/lib/new_relic/agent/instrumentation/async_http.rb
+++ b/lib/new_relic/agent/instrumentation/async_http.rb
@@ -7,7 +7,7 @@
 require_relative 'async_http/prepend'
 
 DependencyDetection.defer do
-  named :'async_http'
+  named :async_http
 
   depends_on do
     defined?(Async::HTTP) && Gem::Version.new(Async::HTTP::VERSION) >= Gem::Version.new('0.59.0')