versions 1.1

CHANGELOG.md

@@ -1,6 +1,10 @@
 # Changelog
 
 
+## v 1.1
+
+* Warning messages included with `query_iterative` outputs.
+
 ## v 1.0.3
 
 * Added DSL grammar for DSL V2.7

dimcli/VERSION.py

@@ -1,7 +1,7 @@
 # !/usr/bin/env python
 #  -*- coding: UTF-8 -*-
 
-__version__ = "1.0.3"  # LATEST? =>  https://pypi.org/project/dimcli/
+__version__ = "1.1"  # LATEST? =>  https://pypi.org/project/dimcli/
 __copyright__ = "CopyRight (C) 2018-2023 by Digital Science"
 __license__ = "MIT"
 __author__ = "Michele Pasin"

dimcli/core/api.py

@@ -200,9 +200,11 @@ def query(self, q, show_results=None, retry=0, verbose=None):
     def query_iterative(self, q, show_results=None, limit=1000, skip=0, pause=1.5, force=False, maxlimit=0, verbose=None, _tot_count_prev_query=0, _warnings_tot=None):       
         """Runs a DSL query and then keep querying until all matching records have been extracted. 
         
-        The API returns a maximum of 1000 records per call. If a DSL query results in more than 1000 matches, it is possible to use pagination to get more results. 
+        The API returns a maximum of 1000 records per call. If a DSL query results in more than 1000 matches, it is possible to use pagination to get more results, up to 50k. 
         
         Iterative querying works by automatically paginating through all records available for a result set. The original query gets turned into a loop that uses the `limit` / `skip` operators until all the results available have been extracted. 
+
+        NOTE If any of the iterative queries produce warning messages, these are aggregated and added to the `_warnings`section of the output data.
         
         Parameters
         ----------
@@ -319,10 +321,11 @@ def query_iterative(self, q, show_results=None, limit=1000, skip=0, pause=1.5, f
             printDebug(f"{skip}-{new_skip} / {tot} ({t}s)")
 
         if res["_warnings"]:
+            warnings = [f"""{x} (iteration: {skip}-{new_skip})""" for x in res["_warnings"]]
             if _warnings_tot:
-                _warnings_tot += res["_warnings"]
+                _warnings_tot += warnings
             else:
-                _warnings_tot = res["_warnings"]
+                _warnings_tot = warnings
 
         if flag_force:
             output = self.query_iterative(q, show_results, limit, new_skip, pause, force, maxlimit, verbose, _tot_count_prev_query, _warnings_tot)                    
@@ -357,7 +360,10 @@ def query_iterative(self, q, show_results=None, limit=1000, skip=0, pause=1.5, f
             result = DslDataset(response_simulation)
             if show_results or (show_results is None and self._show_results):
                 IPython.display.display(result)
-            if verbose: printDebug(f"===\nRecords extracted: {len(output)}")
+            if verbose: 
+                printDebug(f"===\nRecords extracted: {len(output)}")
+                if _warnings_tot: 
+                    printDebug(f"Warnings:  {len(_warnings_tot)}")
             return result
         else:
             return output

docs/_modules/dimcli/core/api.html

@@ -267,9 +267,11 @@ <h1>Source code for dimcli.core.api</h1><div class="highlight"><pre>
 <div class="viewcode-block" id="Dsl.query_iterative"><a class="viewcode-back" href="../../../modules.html#dimcli.core.api.Dsl.query_iterative">[docs]</a>    <span class="k">def</span> <span class="nf">query_iterative</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">q</span><span class="p">,</span> <span class="n">show_results</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">limit</span><span class="o">=</span><span class="mi">1000</span><span class="p">,</span> <span class="n">skip</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">pause</span><span class="o">=</span><span class="mf">1.5</span><span class="p">,</span> <span class="n">force</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">maxlimit</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">verbose</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">_tot_count_prev_query</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">_warnings_tot</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>       
         <span class="sd">&quot;&quot;&quot;Runs a DSL query and then keep querying until all matching records have been extracted. </span>
 <span class="sd">        </span>
-<span class="sd">        The API returns a maximum of 1000 records per call. If a DSL query results in more than 1000 matches, it is possible to use pagination to get more results. </span>
+<span class="sd">        The API returns a maximum of 1000 records per call. If a DSL query results in more than 1000 matches, it is possible to use pagination to get more results, up to 50k. </span>
 <span class="sd">        </span>
 <span class="sd">        Iterative querying works by automatically paginating through all records available for a result set. The original query gets turned into a loop that uses the `limit` / `skip` operators until all the results available have been extracted. </span>
+
+<span class="sd">        NOTE If any of the iterative queries produce warning messages, these are aggregated and added to the `_warnings`section of the output data.</span>
 <span class="sd">        </span>
 <span class="sd">        Parameters</span>
 <span class="sd">        ----------</span>
@@ -386,10 +388,11 @@ <h1>Source code for dimcli.core.api</h1><div class="highlight"><pre>
             <span class="n">printDebug</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">skip</span><span class="si">}</span><span class="s2">-</span><span class="si">{</span><span class="n">new_skip</span><span class="si">}</span><span class="s2"> / </span><span class="si">{</span><span class="n">tot</span><span class="si">}</span><span class="s2"> (</span><span class="si">{</span><span class="n">t</span><span class="si">}</span><span class="s2">s)&quot;</span><span class="p">)</span>
 
         <span class="k">if</span> <span class="n">res</span><span class="p">[</span><span class="s2">&quot;_warnings&quot;</span><span class="p">]:</span>
+            <span class="n">warnings</span> <span class="o">=</span> <span class="p">[</span><span class="sa">f</span><span class="s2">&quot;&quot;&quot;</span><span class="si">{</span><span class="n">x</span><span class="si">}</span><span class="s2"> (iteration: </span><span class="si">{</span><span class="n">skip</span><span class="si">}</span><span class="s2">-</span><span class="si">{</span><span class="n">new_skip</span><span class="si">}</span><span class="s2">)&quot;&quot;&quot;</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">res</span><span class="p">[</span><span class="s2">&quot;_warnings&quot;</span><span class="p">]]</span>
             <span class="k">if</span> <span class="n">_warnings_tot</span><span class="p">:</span>
-                <span class="n">_warnings_tot</span> <span class="o">+=</span> <span class="n">res</span><span class="p">[</span><span class="s2">&quot;_warnings&quot;</span><span class="p">]</span>
+                <span class="n">_warnings_tot</span> <span class="o">+=</span> <span class="n">warnings</span>
             <span class="k">else</span><span class="p">:</span>
-                <span class="n">_warnings_tot</span> <span class="o">=</span> <span class="n">res</span><span class="p">[</span><span class="s2">&quot;_warnings&quot;</span><span class="p">]</span>
+                <span class="n">_warnings_tot</span> <span class="o">=</span> <span class="n">warnings</span>
 
         <span class="k">if</span> <span class="n">flag_force</span><span class="p">:</span>
             <span class="n">output</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">query_iterative</span><span class="p">(</span><span class="n">q</span><span class="p">,</span> <span class="n">show_results</span><span class="p">,</span> <span class="n">limit</span><span class="p">,</span> <span class="n">new_skip</span><span class="p">,</span> <span class="n">pause</span><span class="p">,</span> <span class="n">force</span><span class="p">,</span> <span class="n">maxlimit</span><span class="p">,</span> <span class="n">verbose</span><span class="p">,</span> <span class="n">_tot_count_prev_query</span><span class="p">,</span> <span class="n">_warnings_tot</span><span class="p">)</span>                    
@@ -424,7 +427,10 @@ <h1>Source code for dimcli.core.api</h1><div class="highlight"><pre>
             <span class="n">result</span> <span class="o">=</span> <span class="n">DslDataset</span><span class="p">(</span><span class="n">response_simulation</span><span class="p">)</span>
             <span class="k">if</span> <span class="n">show_results</span> <span class="ow">or</span> <span class="p">(</span><span class="n">show_results</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_show_results</span><span class="p">):</span>
                 <span class="n">IPython</span><span class="o">.</span><span class="n">display</span><span class="o">.</span><span class="n">display</span><span class="p">(</span><span class="n">result</span><span class="p">)</span>
-            <span class="k">if</span> <span class="n">verbose</span><span class="p">:</span> <span class="n">printDebug</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;===</span><span class="se">\n</span><span class="s2">Records extracted: </span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">output</span><span class="p">)</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
+            <span class="k">if</span> <span class="n">verbose</span><span class="p">:</span> 
+                <span class="n">printDebug</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;===</span><span class="se">\n</span><span class="s2">Records extracted: </span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">output</span><span class="p">)</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
+                <span class="k">if</span> <span class="n">_warnings_tot</span><span class="p">:</span> 
+                    <span class="n">printDebug</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Warnings:  </span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">_warnings_tot</span><span class="p">)</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
             <span class="k">return</span> <span class="n">result</span>
         <span class="k">else</span><span class="p">:</span>
             <span class="k">return</span> <span class="n">output</span></div>

docs/modules.html

@@ -270,8 +270,9 @@ <h1>Modules Reference<a class="headerlink" href="#modules-reference" title="Perm
 <dt class="sig sig-object py" id="dimcli.core.api.Dsl.query_iterative">
 <span class="sig-name descname"><span class="pre">query_iterative</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">q</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">show_results</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">limit</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">1000</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">skip</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">0</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">pause</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">1.5</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">force</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">False</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">maxlimit</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">0</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">verbose</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">_tot_count_prev_query</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">0</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">_warnings_tot</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/dimcli/core/api.html#Dsl.query_iterative"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#dimcli.core.api.Dsl.query_iterative" title="Permalink to this definition"></a></dt>
 <dd><p>Runs a DSL query and then keep querying until all matching records have been extracted.</p>
-<p>The API returns a maximum of 1000 records per call. If a DSL query results in more than 1000 matches, it is possible to use pagination to get more results.</p>
+<p>The API returns a maximum of 1000 records per call. If a DSL query results in more than 1000 matches, it is possible to use pagination to get more results, up to 50k.</p>
 <p>Iterative querying works by automatically paginating through all records available for a result set. The original query gets turned into a loop that uses the <cite>limit</cite> / <cite>skip</cite> operators until all the results available have been extracted.</p>
+<p>NOTE If any of the iterative queries produce warning messages, these are aggregated and added to the <a href="#id1"><span class="problematic" id="id2">`</span></a>_warnings`section of the output data.</p>
 <dl class="field-list simple">
 <dt class="field-odd">Parameters</dt>
 <dd class="field-odd"><ul class="simple">
@@ -1454,23 +1455,23 @@ <h1>Modules Reference<a class="headerlink" href="#modules-reference" title="Perm
 &lt;<a class="reference external" href="http://click.pocoo.org/5/api/#click.style">http://click.pocoo.org/5/api/#click.style</a>&gt;
 Styles a text with ANSI styles and returns the new string. By default the styling is self contained which means that at the end of the string a reset code is issued. This can be prevented by passing reset=False.</p>
 <p>This works also with inner click styles eg</p>
-<p><a href="#id1"><span class="problematic" id="id2">``</span></a><a href="#id3"><span class="problematic" id="id4">`</span></a>python
+<p><a href="#id3"><span class="problematic" id="id4">``</span></a><a href="#id5"><span class="problematic" id="id6">`</span></a>python
 uri, title = “<a class="reference external" href="http://example.com">http://example.com</a>”, “My ontology”
 printDebug(click.style(“[%d]” % 1, fg=’blue’) +</p>
 <blockquote>
 <div><p>click.style(uri + ” ==&gt; “, fg=’black’) +
 click.style(title, fg=’red’))</p>
 </div></blockquote>
-<p><a href="#id5"><span class="problematic" id="id6">``</span></a><a href="#id7"><span class="problematic" id="id8">`</span></a></p>
+<p><a href="#id7"><span class="problematic" id="id8">``</span></a><a href="#id9"><span class="problematic" id="id10">`</span></a></p>
 <p>Or even with Colorama</p>
-<p><a href="#id9"><span class="problematic" id="id10">``</span></a>`
+<p><a href="#id11"><span class="problematic" id="id12">``</span></a>`
 from colorama import Fore, Style</p>
 <dl class="simple">
 <dt>printDebug(Fore.BLUE + Style.BRIGHT + “[%d]” % 1 +</dt><dd><p>Style.RESET_ALL + uri + ” ==&gt; ” + Fore.RED + title +
 Style.RESET_ALL)</p>
 </dd>
 </dl>
-<p><a href="#id11"><span class="problematic" id="id12">``</span></a><a href="#id13"><span class="problematic" id="id14">`</span></a></p>
+<p><a href="#id13"><span class="problematic" id="id14">``</span></a><a href="#id15"><span class="problematic" id="id16">`</span></a></p>
 <p>Examples:</p>
 <p>click.echo(click.style(‘Hello World!’, fg=’green’))
 click.echo(click.style(‘ATTENTION!’, blink=True))