525 add general endpoint and job sample rates (#526)

Co-authored-by: Mitch Hartweg <[email protected]>

lib/scout_apm/config.rb

@@ -44,9 +44,11 @@
 #                            instruments listed in this array. Default: []
 # ignore_endpoints         - An array of endpoints to ignore. These are matched as regular expressions. (supercedes 'ignore')
 # ignore_jobs              - An array of job names to ignore.
-# sample_rate              - An integer between 0 and 100. 0 means no traces are sent, 100 means all traces are sent.
+# sample_rate              - Rate to sample entire application. An integer between 0 and 100. 0 means no traces are sent, 100 means all traces are sent.
 # sample_endpoints         - An array of endpoints to sample. These are matched as regular expressions with individual sample rate of 0 to 100.
 # sample_jobs              - An array of job names with individual sample rate of 0 to 100.
+# endpoint_sample_rate     - Rate to sample all endpoints. An integer between 0 and 100. 0 means no traces are sent, 100 means all traces are sent. (supercedes 'sample_rate')
+# job_sample_rate          - Rate to sample all jobs. An integer between 0 and 100. 0 means no traces are sent, 100 means all traces are sent. (supercedes 'sample_rate')
 #
 # Any of these config settings can be set with an environment variable prefixed
 # by SCOUT_ and uppercasing the key: SCOUT_LOG_LEVEL for instance.
@@ -93,6 +95,8 @@ class Config
         'sample_rate',
         'sample_endpoints',
         'sample_jobs',
+        'endpoint_sample_rate',
+        'job_sample_rate',
         'scm_subdirectory',
         'start_resque_server_instrument',
         'ssl_cert_file',
@@ -209,6 +213,8 @@ def coerce(val)
       'sample_rate' => IntegerCoercion.new,
       'sample_endpoints' => JsonCoercion.new,
       'sample_jobs' => JsonCoercion.new,
+      'endpoint_sample_rate' => IntegerCoercion.new,
+      'job_sample_rate' => IntegerCoercion.new,
       'start_resque_server_instrument' => BooleanCoercion.new,
       'timeline_traces' => BooleanCoercion.new,
       'auto_instruments' => BooleanCoercion.new,
@@ -331,6 +337,8 @@ class ConfigDefaults
         'sample_rate'                          => 100,
         'sample_endpoints'                     => [],
         'sample_jobs'                          => [],
+        'endpoint_sample_rate'                 => 100,
+        'job_sample_rate'                      => 100,
         'start_resque_server_instrument'       => true, # still only starts if Resque is detected
         'collect_remote_ip'                    => true,
         'record_queue_time'                    => true,

lib/scout_apm/sampling.rb

@@ -10,27 +10,31 @@ def initialize(config)
       # for now still support old config key ('ignore') for backwards compatibility
       @ignore_endpoints = config.value('ignore').present? ? config.value('ignore') : config.value('ignore_endpoints')
       @sample_endpoints = individual_sample_to_hash(config.value('sample_endpoints'))
+      @endpoint_sample_rate = config.value('endpoint_sample_rate')
 
       @ignore_jobs = config.value('ignore_jobs')
       @sample_jobs = individual_sample_to_hash(config.value('sample_jobs'))
+      @job_sample_rate = config.value('job_sample_rate')
 
-      logger.info("Sampling initialized with config: global_sample_rate: #{@global_sample_rate}, sample_endpoints: #{@sample_endpoints}, ignore_endpoints: #{@ignore_endpoints}, sample_jobs: #{@sample_jobs}, ignore_jobs: #{@ignore_jobs}")
+      logger.info("Sampling initialized with config: global_sample_rate: #{@global_sample_rate}, endpoint_sample_rate: #{@endpoint_sample_rate}, sample_endpoints: #{@sample_endpoints}, ignore_endpoints: #{@ignore_endpoints}, job_sample_rate: #@job_sample_rate}, sample_jobs: #{@sample_jobs}, ignore_jobs: #{@ignore_jobs}")
     end
 
     def drop_request?(transaction)
-      # job or endpoint?
-      # check if request should be sampled first
-      # Individual sample rate always takes precedence over global sample rate
+      # Individual endpoint/job sampling takes precedence over ignoring.
+      # Individual endpoint/job sample rate always takes precedence over general endpoint/job rate.
+      # General endpoint/job rate always takes precedence over global sample rate
       if transaction.job?
         job_name = transaction.layer_finder.job.name
         rate = job_sample_rate(job_name)
         return sample?(rate) unless rate.nil?
         return true if ignore_job?(job_name)
+        return sample?(@job_sample_rate) unless @job_sample_rate.nil?
       elsif transaction.web?
         uri = transaction.annotations[:uri]
         rate = web_sample_rate(uri)
         return sample?(rate) unless rate.nil?
         return true if ignore_uri?(uri)
+        return sample?(@endpoint_sample_rate) unless @endpoint_sample_rate.nil?
       end
 
       # global sample check

test/unit/sampling_test.rb

@@ -91,7 +91,6 @@ def test_web_request_individual_sampling
       transaction = FakeTrackedRequest.new_web_request('/faz/bap')
       assert_equal false, sampling.drop_request?(transaction)
 
-
       transaction = FakeTrackedRequest.new_web_request('/foo/far')
       sampling.stub(:rand, 0.01) do
         assert_equal false, sampling.drop_request?(transaction)
@@ -103,6 +102,29 @@ def test_web_request_individual_sampling
       end
     end
 
+    def test_web_reqeust_general_sampling
+      config = FakeConfigOverlay.new(@individual_config.values.merge({'endpoint_sample_rate' => 80}))
+      sampling = ScoutApm::Sampling.new(config)
+
+      transaction = FakeTrackedRequest.new_web_request('/foo/far')
+      transaction2 = FakeTrackedRequest.new_web_request('/ooo/oar')
+      # /foo/far sampled at 50 specifically, /ooo/oar caught by general endpoint rate of 80
+      sampling.stub(:rand, 0.01) do
+        assert_equal false, sampling.drop_request?(transaction)
+        assert_equal false, sampling.drop_request?(transaction2)
+      end
+
+      sampling.stub(:rand, 0.70) do
+        assert_equal true, sampling.drop_request?(transaction)
+        assert_equal false, sampling.drop_request?(transaction2)
+      end
+
+      sampling.stub(:rand, 0.99) do
+        assert_equal true, sampling.drop_request?(transaction)
+        assert_equal true, sampling.drop_request?(transaction2)
+      end
+    end
+
     def test_web_request_with_global_sampling
       config = FakeConfigOverlay.new(@individual_config.values.merge({'sample_rate' => 20}))
       sampling = ScoutApm::Sampling.new(config)
@@ -147,6 +169,29 @@ def test_job_request_individual_sampling
       end
     end
 
+    def test_job_general_sampling
+      config = FakeConfigOverlay.new(@individual_config.values.merge({'job_sample_rate' => 80}))
+      sampling = ScoutApm::Sampling.new(config)
+
+      transaction = FakeTrackedRequest.new_job_request('joba')
+      transaction2 = FakeTrackedRequest.new_job_request('jobz')
+      # joba sampled at 50 specifically, jobz caught by general job rate of 80
+      sampling.stub(:rand, 0.01) do
+        assert_equal false, sampling.drop_request?(transaction)
+        assert_equal false, sampling.drop_request?(transaction2)
+      end
+
+      sampling.stub(:rand, 0.70) do
+        assert_equal true, sampling.drop_request?(transaction)
+        assert_equal false, sampling.drop_request?(transaction2)
+      end
+
+      sampling.stub(:rand, 0.99) do
+        assert_equal true, sampling.drop_request?(transaction)
+        assert_equal true, sampling.drop_request?(transaction2)
+      end
+    end
+
     def test_job_request_global_sampling
       config = FakeConfigOverlay.new(@individual_config.values.merge({'sample_rate' => 20}))
       sampling = ScoutApm::Sampling.new(config)