From 761152babe23af6db20297b98c89f2398c750f1b Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Tue, 11 Feb 2020 15:57:11 +0000 Subject: [PATCH 01/68] appveyor: first version --- .appveyor.yml | 112 +++++++++++++++++++++++++++ appveyor-test.py | 66 ++++++++++++++++ appveyor/.appveyor.yml.example-full | 113 ++++++++++++++++++++++++++++ appveyor/do.py | 104 +++++++++++++++++++++++++ 4 files changed, 395 insertions(+) create mode 100644 .appveyor.yml create mode 100644 appveyor-test.py create mode 100644 appveyor/.appveyor.yml.example-full create mode 100644 appveyor/do.py diff --git a/.appveyor.yml b/.appveyor.yml new file mode 100644 index 0000000..b336a73 --- /dev/null +++ b/.appveyor.yml @@ -0,0 +1,112 @@ +# .appveyor.yml for testing EPICS Base ci-scripts +# (see: https://github.com/epics-base/ci-scripts) + +# Note: +# Paths to scripts are different in this test configuration +# (your module has one more directory level: .ci) + +# Ralph Lange +# Copyright (c) 2020 ITER Organization + + +#---------------------------------# +# repository cloning # +#---------------------------------# + +# Called at very beginning, before repo cloning +init: + # Set autocrlf to make batch files work + - git config --global core.autocrlf true + +# Set clone depth (do not fetch complete history) +clone_depth: 50 + +# Skipping commits affecting only specific files +skip_commits: + files: + - 'documentation/*' + - 'templates/*' + - '**/*.html' + - '**/*.md' + +#---------------------------------# +# build matrix configuration # +#---------------------------------# + +# Build Configurations: dll/static, regular/debug +configuration: + - dynamic + - static + - dynamic-debug + - static-debug + +# Environment variables: compiler toolchain +environment: + matrix: + - CC: mingw + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 + - CC: vs2019 + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 + - CC: vs2017 + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 + - CC: vs2015 + - CC: vs2013 + - CC: vs2012 + - CC: vs2010 + +# Platform: architecture +platform: +# - x86 + - x64 + +# Matrix configuration: allow specific failing jobs +matrix: + exclude: + # VS2010 Express installs don't have the 64 bit compiler + - platform: x64 + CC: vs2010 + # Exclude to reduce total job runtime + # skip 64-bit for older and 32-bit for newer + - platform: x64 + CC: vs2012 + - platform: x86 + CC: mingw + - platform: x86 + CC: vs2019 + - platform: x86 + CC: vs2017 + + +#---------------------------------# +# building & testing # +#---------------------------------# + +build_script: + - cmd: python appveyor-test.py + + +#---------------------------------# +# debugging # +#---------------------------------# + +## if you want to connect by remote desktop to a failed build, uncomment these lines +## note that you will need to connect within the usual build timeout limit (60 minutes) +## so you may want to adjust the build matrix above to just build the one of interest + +#on_failure: +# - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) +# - ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) + + +#---------------------------------# +# notifications # +#---------------------------------# + +notifications: + +# - provider: Email +# to: +# - core-talk@aps.anl.gov +# on_build_success: false + +# - provider: GitHubPullRequest diff --git a/appveyor-test.py b/appveyor-test.py new file mode 100644 index 0000000..7828a5f --- /dev/null +++ b/appveyor-test.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +"""Module ci-scripts AppVeyor unit tests +""" + +# SET=test00 in .appveyor.yml runs the tests in this script +# all other jobs are started as compile jobs + +import sys, os +import unittest + +sys.path.append('appveyor') +import do + +class TestSourceSet(unittest.TestCase): + + def setUp(self): + os.environ['SETUP_PATH'] = '.:appveyor' + if 'BASE' in os.environ: + del os.environ['BASE'] + do.clear_lists() + + def test_EmptySetupDirsPath(self): + del os.environ['SETUP_PATH'] + try: + do.source_set('test01') + except NameError: + return + self.fail('source_set did not throw on empty SETUP_DIRS') + + def test_InvalidSetupName(self): + try: + do.source_set('xxdoesnotexistxx') + except NameError: + return + self.fail('source_set did not throw on invalid file name') + + def test_ValidSetupName(self): + do.source_set('test01') + self.assertEqual(do.setup['BASE'], '7.0', 'BASE was not set to \'7.0\'') + + def test_SetupDoesNotOverridePreset(self): + os.environ['BASE'] = 'foo' + do.source_set('test01') + self.assertEqual(do.setup['BASE'], 'foo', + 'Preset BASE was overridden by test01 setup (expected \'foo\' got {0})' + .format(do.setup['BASE'])) + + def test_IncludeSetupFirstSetWins(self): + do.source_set('test02') + self.assertEqual(do.setup['BASE'], 'foo', + 'BASE set in test02 was overridden by test01 setup (expected \'foo\' got {0})' + .format(do.setup['BASE'])) + self.assertEqual(do.setup['FOO'], 'bar', 'Setting of single word does not work') + self.assertEqual(do.setup['FOO2'], 'bar bar2', 'Setting of multiple words does not work') + self.assertEqual(do.setup['FOO3'], 'bar bar2', 'Indented setting of multiple words does not work') + self.assertEqual(do.setup['SNCSEQ'], 'R2-2-7', 'Setup test01 was not included') + + + + + + +if __name__ == "__main__": + suite = unittest.TestLoader().loadTestsFromTestCase(TestSourceSet) + unittest.TextTestRunner(verbosity=2).run(suite) +# unittest.main() diff --git a/appveyor/.appveyor.yml.example-full b/appveyor/.appveyor.yml.example-full new file mode 100644 index 0000000..662edd2 --- /dev/null +++ b/appveyor/.appveyor.yml.example-full @@ -0,0 +1,113 @@ +# .appveyor.yml for use with EPICS Base ci-scripts +# (see: https://github.com/epics-base/ci-scripts) + +# This is YAML - indentation levels are crucial + +#---------------------------------# +# repository cloning # +#---------------------------------# + +# Called at very beginning, before repo cloning +init: + # Set autocrlf to make batch files work + - git config --global core.autocrlf true + +# Set clone depth (do not fetch complete history) +clone_depth: 50 + +# Skipping commits affecting only specific files +skip_commits: + files: + - 'documentation/*' + - 'templates/*' + - '**/*.html' + - '**/*.md' + +#---------------------------------# +# build matrix configuration # +#---------------------------------# + +# Build Configurations: dll/static, regular/debug +configuration: + - dynamic + - static + - dynamic-debug + - static-debug + +# Environment variables: compiler toolchain +environment: + matrix: + - CC: mingw + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 + - CC: vs2019 + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 + - CC: vs2017 + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 + - CC: vs2015 + - CC: vs2013 + - CC: vs2012 + - CC: vs2010 + +# Platform: architecture +platform: + - x86 + - x64 + +# Matrix configuration: allow specific failing jobs +matrix: + exclude: + # VS2010 Express installs don't have the 64 bit compiler + - platform: x64 + CC: vs2010 + # Exclude to reduce total job runtime + # skip 64-bit for older and 32-bit for newer + - platform: x64 + CC: vs2012 + - platform: x86 + CC: mingw + - platform: x86 + CC: vs2019 + - platform: x86 + CC: vs2017 + +#---------------------------------# +# building & testing # +#---------------------------------# + +install: + - cmd: git submodule update --init --recursive + - cmd: python .ci/appveyor/do.py prepare + +build_script: + - cmd: python .ci/appveyor/do.py build + +test_script: + - cmd: python .ci/appveyor/do.py test + +on_finish: + - ps: Get-ChildItem *.tap -Recurse -Force | % { Push-AppveyorArtifact $_.FullName -FileName $_.Name } + +#---------------------------------# +# debugging # +#---------------------------------# + +## if you want to connect by remote desktop to a failed build, uncomment these lines +## note that you will need to connect within the usual build timeout limit (60 minutes) +## so you may want to adjust the build matrix above to just build the one of interest + +#on_failure: +# - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) +# - ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) + +#---------------------------------# +# notifications # +#---------------------------------# + +notifications: + + - provider: Email + to: + - me@example.com + on_build_success: false + + - provider: GitHubPullRequest diff --git a/appveyor/do.py b/appveyor/do.py new file mode 100644 index 0000000..b07a21b --- /dev/null +++ b/appveyor/do.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python +"""Windows (AppVeyor) ci build script +""" + +import sys, os +import logging +import subprocess as SP +import distutils.util +from glob import glob + +#logging.basicConfig(level=logging.DEBUG) + +# Setup ANSI Colors +ANSI_RED = "\033[31;1m" +ANSI_GREEN = "\033[32;1m" +ANSI_YELLOW = "\033[33;1m" +ANSI_BLUE = "\033[34;1m" +ANSI_RESET = "\033[0m" +ANSI_CLEAR = "\033[0K" + +seen_setups = [] +setup = {} + +# Used from unittests +def clear_lists(): + del seen_setups[:] + setup.clear() + +# source_set(setup) +# +# Source a settings file (extension .set) found in the setup_dirs path +# May be called recursively (from within a setup file) +def source_set(args): + found = False + + setup_dirs = os.getenv('SETUP_PATH', "").replace(':', ' ').split() + if len(setup_dirs) == 0: + raise NameError("{0}Search path for setup files (SETUP_PATH) is empty{1}".format(ANSI_RED,ANSI_RESET)) + + for set_dir in setup_dirs: + set_file = os.path.join(set_dir, args) + ".set" + + if set_file in seen_setups: + print("Ignoring already included setup file {0}".format(set_file)) + return + + if os.path.isfile(set_file): + seen_setups.append(set_file) + print("Loading setup file {0}".format(set_file)) + with open(set_file) as fp: + for line in fp: + logging.debug('Next line: {0}'.format(line.strip())) + if not line.strip() or line.strip()[0] == '#': + continue + if line.startswith("include"): + logging.debug('Found an include, reading {0}'.format(line.split()[1])) + source_set(line.split()[1]) + continue + assign = line.replace('"', '').strip().split("=", 1) + logging.debug('Interpreting as assignment') + if assign[0] not in setup: + setup[assign[0]] = os.getenv(assign[0], "") + if not setup[assign[0]].strip(): + logging.debug('Doing assignment: {0} = {1}'.format(assign[0], assign[1])) + setup[assign[0]] = assign[1] + found = True + break + + if not found: + raise NameError("{0}Setup file {1} does not exist in SETUP_PATH search path ({2}){3}" + .format(ANSI_RED,set_file,setup_dirs,ANSI_RESET)) + +def prepare(args): + print(sys.version) + print('PYTHONPATH') + for dname in sys.path: + print(' ', dname) + print('platform = ', distutils.util.get_platform()) + + print('{0}Loading setup files{1}'.format(ANSI_YELLOW, ANSI_RESET)) + source_set(default) + if 'SET' in os.environ: + source_set(os.environ['SET']) + + print('Installing dependencies') + +def build(args): + print('Building the module') + +def test(args): + print('Running the tests') + +actions = { + 'prepare': prepare, + 'build': build, + 'test': test, +} + +if __name__=='__main__': + args = sys.argv[1:] + while len(args)>0: + name = args.pop(0) + print('IN', name, 'with', args) + actions[name](args) From b15d9bb62eb0d1fdfcbcb3b07bebb60e53d3259b Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Tue, 11 Feb 2020 15:57:11 +0000 Subject: [PATCH 02/68] appveyor: first version (source_set, update_release_local) --- appveyor-test.py | 60 +++++++++++++++++++++++++++++++++++--- appveyor/do.py | 75 +++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 127 insertions(+), 8 deletions(-) diff --git a/appveyor-test.py b/appveyor-test.py index 7828a5f..4154d22 100644 --- a/appveyor-test.py +++ b/appveyor-test.py @@ -5,7 +5,7 @@ # SET=test00 in .appveyor.yml runs the tests in this script # all other jobs are started as compile jobs -import sys, os +import sys, os, fileinput import unittest sys.path.append('appveyor') @@ -56,11 +56,63 @@ def test_IncludeSetupFirstSetWins(self): self.assertEqual(do.setup['SNCSEQ'], 'R2-2-7', 'Setup test01 was not included') +class TestUpdateReleaseLocal(unittest.TestCase): + release_local = os.path.join(do.cachedir, 'RELEASE.local') + + def setUp(self): + if os.path.exists(self.release_local): + os.remove(self.release_local) + + def test_SetModule(self): + do.update_release_local('MOD1', '/foo/bar') + found = 0 + for line in fileinput.input(self.release_local, inplace=1): + if 'MOD1=' in line: + self.assertEqual(line.strip(), 'MOD1=/foo/bar', 'MOD1 not set correctly') + found += 1 + fileinput.close() + self.assertEqual(found, 1, 'MOD1 not written once to RELEASE.local (found {0})'.format(found)) + + def test_SetBaseAndMultipleModules(self): + do.update_release_local('EPICS_BASE', '/bar/foo') + do.update_release_local('MOD1', '/foo/bar') + do.update_release_local('MOD2', '/foo/bar2') + do.update_release_local('MOD1', '/foo/bar1') + foundmod1 = 0 + foundmod2 = 0 + foundbase = 0 + for line in fileinput.input(self.release_local, inplace=1): + if 'MOD1=' in line: + self.assertEqual(line.strip(), 'MOD1=/foo/bar1', + 'MOD1 not set correctly (expected \'MOD1=/foo/bar1\' found \'{0}\')' + .format(line)) + foundmod1 += 1 + foundmod1at = fileinput.filelineno() + if 'MOD2=' in line: + self.assertEqual(line.strip(), 'MOD2=/foo/bar2', + 'MOD2 not set correctly (expected \'MOD2=/foo/bar2\' found \'{0}\')' + .format(line)) + foundmod2 += 1 + foundmod2at = fileinput.filelineno() + if 'EPICS_BASE=' in line: + self.assertEqual(line.strip(), 'EPICS_BASE=/bar/foo', + 'EPICS_BASE not set correctly (expected \'EPICS_BASE=/bar/foo\' found \'{0}\')' + .format(line)) + foundbase += 1 + foundbaseat = fileinput.filelineno() + fileinput.close() + self.assertEqual(foundmod1, 1, 'MOD1 does not appear once in RELEASE.local (found {0})'.format(foundmod1)) + self.assertEqual(foundmod2, 1, 'MOD2 does not appear once in RELEASE.local (found {0})'.format(foundmod2)) + self.assertEqual(foundbase, 1, 'EPICS_BASE does not appear once in RELEASE.local (found {0})'.format(foundbase)) + self.assertGreater(foundbaseat, foundmod2at, + 'EPICS_BASE (line {0}) appears before MOD2 (line {1})'.format(foundbaseat, foundmod2at)) + self.assertGreater(foundmod2at, foundmod1at, + 'MOD2 (line {0}) appears before MOD1 (line {1})'.format(foundmod2at, foundmod1at)) if __name__ == "__main__": - suite = unittest.TestLoader().loadTestsFromTestCase(TestSourceSet) - unittest.TextTestRunner(verbosity=2).run(suite) -# unittest.main() +# suite = unittest.TestLoader().loadTestsFromTestCase(TestUpdateReleaseLocal) +# unittest.TextTestRunner(verbosity=2).run(suite) + unittest.main() diff --git a/appveyor/do.py b/appveyor/do.py index b07a21b..052fb61 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -2,11 +2,12 @@ """Windows (AppVeyor) ci build script """ -import sys, os +from __future__ import print_function + +import sys, os, fileinput import logging import subprocess as SP import distutils.util -from glob import glob #logging.basicConfig(level=logging.DEBUG) @@ -20,6 +21,12 @@ seen_setups = [] setup = {} +if 'HomeDrive' in os.environ: + cachedir = os.path.join(os.getenv('HomeDrive'), os.getenv('HomePath'), '.cache') +elif 'HOME' in os.environ: + cachedir = os.path.join(os.getenv('HOME'), '.cache') +else: + cachedir = os.path.join('.', '.cache') # Used from unittests def clear_lists(): @@ -30,7 +37,7 @@ def clear_lists(): # # Source a settings file (extension .set) found in the setup_dirs path # May be called recursively (from within a setup file) -def source_set(args): +def source_set(set): found = False setup_dirs = os.getenv('SETUP_PATH', "").replace(':', ' ').split() @@ -38,7 +45,7 @@ def source_set(args): raise NameError("{0}Search path for setup files (SETUP_PATH) is empty{1}".format(ANSI_RED,ANSI_RESET)) for set_dir in setup_dirs: - set_file = os.path.join(set_dir, args) + ".set" + set_file = os.path.join(set_dir, set) + ".set" if set_file in seen_setups: print("Ignoring already included setup file {0}".format(set_file)) @@ -70,6 +77,66 @@ def source_set(args): raise NameError("{0}Setup file {1} does not exist in SETUP_PATH search path ({2}){3}" .format(ANSI_RED,set_file,setup_dirs,ANSI_RESET)) +# update_release_local(var, place) +# var name of the variable to set in RELEASE.local +# place place (absolute path) of where variable should point to +# +# Manipulate RELEASE.local in the cache location: +# - replace "$var=$place" line if it exists and has changed +# - otherwise add "$var=$place" line and possibly move EPICS_BASE=... line to the end +def update_release_local(var, place): + release_local = os.path.join(cachedir, 'RELEASE.local') + updated_line = '{0}={1}'.format(var, place) + + if not os.path.exists(release_local): + logging.debug('RELEASE.local does not exist, creating it') + try: + os.makedirs(cachedir) + except: + pass + fout = open(release_local, 'w') + fout.close() + base_line = '' + found = False + logging.debug('Opening RELEASE.local for adding {0}={1}'.format(var, place)) + for line in fileinput.input(release_local, inplace=1): + if 'EPICS_BASE=' in line: + logging.debug('Found EPICS_BASE line \'{0}\', not writing it'.format(line.strip())) + base_line = line.strip() + continue + elif '{0}='.format(var) in line: + logging.debug('Found \'{0}=\' line, replacing'.format(var)) + found = True + line = updated_line + logging.debug('Writing line to RELEASE.local: \'{0}\''.format(line)) + print(line) + fileinput.close() + fout = open(release_local,"a") + if not found: + logging.debug('Adding new definition: \'{0}\''.format(updated_line)) + print(updated_line, file=fout) + if base_line: + logging.debug('Writing EPICS_BASE line: \'{0}\''.format(base_line)) + print(base_line, file=fout) + fout.close() + +# add_dependency(dep, tag) +# +# Add a dependency to the cache area: +# - check out (recursive if configured) in the CACHE area unless it already exists and the +# required commit has been built +# - Defaults: +# $dep_DIRNAME = lower case ($dep) +# $dep_REPONAME = lower case ($dep) +# $dep_REPOURL = GitHub / $dep_REPOOWNER (or $REPOOWNER or epics-modules) / $dep_REPONAME .git +# $dep_VARNAME = $dep +# $dep_DEPTH = 5 +# $dep_RECURSIVE = 1/YES (0/NO to for a flat clone) +# - Add $dep_VARNAME line to the RELEASE.local file in the cache area (unless already there) +# - Add full path to $modules_to_compile +def add_dependency(dep, tag): + pass + def prepare(args): print(sys.version) print('PYTHONPATH') From 139b491614fd691d703e1e1591bcaec02ebb82d2 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Mon, 17 Feb 2020 09:52:19 +0100 Subject: [PATCH 03/68] appveyor: improve tests (capture stdout; use dictionaries) --- appveyor-test.py | 68 ++++++++++++++++++++++++++++++------------------ 1 file changed, 42 insertions(+), 26 deletions(-) diff --git a/appveyor-test.py b/appveyor-test.py index 4154d22..42478ea 100644 --- a/appveyor-test.py +++ b/appveyor-test.py @@ -8,6 +8,14 @@ import sys, os, fileinput import unittest +def getStringIO(): + if (sys.version_info > (3, 0)): + import io + return io.StringIO() + else: + import StringIO + return StringIO.StringIO() + sys.path.append('appveyor') import do @@ -21,32 +29,33 @@ def setUp(self): def test_EmptySetupDirsPath(self): del os.environ['SETUP_PATH'] - try: - do.source_set('test01') - except NameError: - return - self.fail('source_set did not throw on empty SETUP_DIRS') + self.assertRaisesRegexp(NameError, '\(SETUP_PATH\) is empty', do.source_set, 'test01') def test_InvalidSetupName(self): - try: - do.source_set('xxdoesnotexistxx') - except NameError: - return - self.fail('source_set did not throw on invalid file name') + self.assertRaisesRegexp(NameError, 'does not exist in SETUP_PATH', do.source_set, 'xxdoesnotexistxx') def test_ValidSetupName(self): + capturedOutput = getStringIO() + sys.stdout = capturedOutput do.source_set('test01') + sys.stdout = sys.__stdout__ self.assertEqual(do.setup['BASE'], '7.0', 'BASE was not set to \'7.0\'') def test_SetupDoesNotOverridePreset(self): os.environ['BASE'] = 'foo' + capturedOutput = getStringIO() + sys.stdout = capturedOutput do.source_set('test01') + sys.stdout = sys.__stdout__ self.assertEqual(do.setup['BASE'], 'foo', 'Preset BASE was overridden by test01 setup (expected \'foo\' got {0})' .format(do.setup['BASE'])) def test_IncludeSetupFirstSetWins(self): + capturedOutput = getStringIO() + sys.stdout = capturedOutput do.source_set('test02') + sys.stdout = sys.__stdout__ self.assertEqual(do.setup['BASE'], 'foo', 'BASE set in test02 was overridden by test01 setup (expected \'foo\' got {0})' .format(do.setup['BASE'])) @@ -55,6 +64,12 @@ def test_IncludeSetupFirstSetWins(self): self.assertEqual(do.setup['FOO3'], 'bar bar2', 'Indented setting of multiple words does not work') self.assertEqual(do.setup['SNCSEQ'], 'R2-2-7', 'Setup test01 was not included') + def test_DoubleIncludeGetsIgnored(self): + capturedOutput = getStringIO() + sys.stdout = capturedOutput + do.source_set('test03') + sys.stdout = sys.__stdout__ + self.assertRegexpMatches(capturedOutput.getvalue(), 'Ignoring already included setup file') class TestUpdateReleaseLocal(unittest.TestCase): @@ -79,40 +94,41 @@ def test_SetBaseAndMultipleModules(self): do.update_release_local('MOD1', '/foo/bar') do.update_release_local('MOD2', '/foo/bar2') do.update_release_local('MOD1', '/foo/bar1') - foundmod1 = 0 - foundmod2 = 0 - foundbase = 0 + found = {} + foundat = {} for line in fileinput.input(self.release_local, inplace=1): if 'MOD1=' in line: self.assertEqual(line.strip(), 'MOD1=/foo/bar1', 'MOD1 not set correctly (expected \'MOD1=/foo/bar1\' found \'{0}\')' .format(line)) - foundmod1 += 1 - foundmod1at = fileinput.filelineno() + found['mod1'] += 1 + foundat['mod1'] = fileinput.filelineno() if 'MOD2=' in line: self.assertEqual(line.strip(), 'MOD2=/foo/bar2', 'MOD2 not set correctly (expected \'MOD2=/foo/bar2\' found \'{0}\')' .format(line)) - foundmod2 += 1 - foundmod2at = fileinput.filelineno() + found['mod2'] += 1 + foundat['mod2'] = fileinput.filelineno() if 'EPICS_BASE=' in line: self.assertEqual(line.strip(), 'EPICS_BASE=/bar/foo', 'EPICS_BASE not set correctly (expected \'EPICS_BASE=/bar/foo\' found \'{0}\')' .format(line)) - foundbase += 1 - foundbaseat = fileinput.filelineno() + found['base'] += 1 + foundat['base'] = fileinput.filelineno() fileinput.close() - self.assertEqual(foundmod1, 1, 'MOD1 does not appear once in RELEASE.local (found {0})'.format(foundmod1)) - self.assertEqual(foundmod2, 1, 'MOD2 does not appear once in RELEASE.local (found {0})'.format(foundmod2)) - self.assertEqual(foundbase, 1, 'EPICS_BASE does not appear once in RELEASE.local (found {0})'.format(foundbase)) - self.assertGreater(foundbaseat, foundmod2at, - 'EPICS_BASE (line {0}) appears before MOD2 (line {1})'.format(foundbaseat, foundmod2at)) - self.assertGreater(foundmod2at, foundmod1at, - 'MOD2 (line {0}) appears before MOD1 (line {1})'.format(foundmod2at, foundmod1at)) + self.assertEqual(found['mod1'], 1, 'MOD1 does not appear once in RELEASE.local (found {0})'.format(found['mod1'])) + self.assertEqual(found['mod2'], 1, 'MOD2 does not appear once in RELEASE.local (found {0})'.format(found['mod2'])) + self.assertEqual(found['base'], 1, 'EPICS_BASE does not appear once in RELEASE.local (found {0})'.format(found['base'])) + self.assertGreater(foundat['base'], foundat['mod2'], + 'EPICS_BASE (line {0}) appears before MOD2 (line {1})'.format(foundat['base'], foundat['mod2'])) + self.assertGreater(foundat['mod2'], foundat['mod1'], + 'MOD2 (line {0}) appears before MOD1 (line {1})'.format(foundat['mod2'], foundat['mod1'])) + if __name__ == "__main__": +# suite = unittest.TestLoader().loadTestsFromTestCase(TestSourceSet) # suite = unittest.TestLoader().loadTestsFromTestCase(TestUpdateReleaseLocal) # unittest.TextTestRunner(verbosity=2).run(suite) unittest.main() From 355a5c2fb772a156dee0d5735ad4524513e2855b Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Wed, 19 Feb 2020 09:30:35 +0100 Subject: [PATCH 04/68] appveyor: fix logging in do.py --- appveyor/do.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/appveyor/do.py b/appveyor/do.py index 052fb61..2817315 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -9,6 +9,7 @@ import subprocess as SP import distutils.util +logger = logging.getLogger(__name__) #logging.basicConfig(level=logging.DEBUG) # Setup ANSI Colors @@ -56,19 +57,18 @@ def source_set(set): print("Loading setup file {0}".format(set_file)) with open(set_file) as fp: for line in fp: - logging.debug('Next line: {0}'.format(line.strip())) + logger.debug('Next line: %s', line.strip()) if not line.strip() or line.strip()[0] == '#': continue if line.startswith("include"): - logging.debug('Found an include, reading {0}'.format(line.split()[1])) + logger.debug('Found an include, reading %s', line.split()[1]) source_set(line.split()[1]) continue assign = line.replace('"', '').strip().split("=", 1) - logging.debug('Interpreting as assignment') - if assign[0] not in setup: - setup[assign[0]] = os.getenv(assign[0], "") + logger.debug('Interpreting as assignment') + setup.setdefault(assign[0], os.getenv(assign[0], "")) if not setup[assign[0]].strip(): - logging.debug('Doing assignment: {0} = {1}'.format(assign[0], assign[1])) + logger.debug('Doing assignment: %s = %s', assign[0], assign[1]) setup[assign[0]] = assign[1] found = True break @@ -89,7 +89,7 @@ def update_release_local(var, place): updated_line = '{0}={1}'.format(var, place) if not os.path.exists(release_local): - logging.debug('RELEASE.local does not exist, creating it') + logger.debug('RELEASE.local does not exist, creating it') try: os.makedirs(cachedir) except: @@ -98,25 +98,25 @@ def update_release_local(var, place): fout.close() base_line = '' found = False - logging.debug('Opening RELEASE.local for adding {0}={1}'.format(var, place)) + logger.debug("Opening RELEASE.local for adding '%s'", updated_line) for line in fileinput.input(release_local, inplace=1): if 'EPICS_BASE=' in line: - logging.debug('Found EPICS_BASE line \'{0}\', not writing it'.format(line.strip())) + logger.debug("Found EPICS_BASE line '%s', not writing it", base_line) base_line = line.strip() continue elif '{0}='.format(var) in line: - logging.debug('Found \'{0}=\' line, replacing'.format(var)) + logger.debug("Found '%s=' line, replacing", var) found = True line = updated_line - logging.debug('Writing line to RELEASE.local: \'{0}\''.format(line)) + logger.debug("Writing line to RELEASE.local: '%s'", outputline) print(line) fileinput.close() fout = open(release_local,"a") if not found: - logging.debug('Adding new definition: \'{0}\''.format(updated_line)) + logger.debug("Adding new definition: '%s'", updated_line) print(updated_line, file=fout) if base_line: - logging.debug('Writing EPICS_BASE line: \'{0}\''.format(base_line)) + logger.debug("Writing EPICS_BASE line: '%s'", base_line) print(base_line, file=fout) fout.close() From cd0becff0664f0e9997d93115995dadeaa9c9b51 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Mon, 17 Feb 2020 10:04:02 +0100 Subject: [PATCH 05/68] appveyor: add add_dependency() --- appveyor-test.py | 89 +++++++++++++++++++++++++++++++---- appveyor/do.py | 119 +++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 191 insertions(+), 17 deletions(-) diff --git a/appveyor-test.py b/appveyor-test.py index 42478ea..a9643c5 100644 --- a/appveyor-test.py +++ b/appveyor-test.py @@ -5,9 +5,19 @@ # SET=test00 in .appveyor.yml runs the tests in this script # all other jobs are started as compile jobs -import sys, os, fileinput +from __future__ import print_function + +import sys, os, shutil, fileinput +import re import unittest +def find_in_file(regex, filename): + file = open (filename, "r") + for line in file: + if re.search(regex, line): + return True + return False + def getStringIO(): if (sys.version_info > (3, 0)): import io @@ -19,6 +29,9 @@ def getStringIO(): sys.path.append('appveyor') import do +# we're working with tags (detached heads) a lot: suppress advice +do.call_git(['config', '--global', 'advice.detachedHead', 'false']) + class TestSourceSet(unittest.TestCase): def setUp(self): @@ -101,34 +114,94 @@ def test_SetBaseAndMultipleModules(self): self.assertEqual(line.strip(), 'MOD1=/foo/bar1', 'MOD1 not set correctly (expected \'MOD1=/foo/bar1\' found \'{0}\')' .format(line)) - found['mod1'] += 1 + if 'mod1' in found: + found['mod1'] += 1 + else: + found['mod1'] = 1 foundat['mod1'] = fileinput.filelineno() if 'MOD2=' in line: self.assertEqual(line.strip(), 'MOD2=/foo/bar2', 'MOD2 not set correctly (expected \'MOD2=/foo/bar2\' found \'{0}\')' .format(line)) - found['mod2'] += 1 + if 'mod2' in found: + found['mod2'] += 1 + else: + found['mod2'] = 1 foundat['mod2'] = fileinput.filelineno() if 'EPICS_BASE=' in line: self.assertEqual(line.strip(), 'EPICS_BASE=/bar/foo', 'EPICS_BASE not set correctly (expected \'EPICS_BASE=/bar/foo\' found \'{0}\')' .format(line)) - found['base'] += 1 + if 'base' in found: + found['base'] += 1 + else: + found['base'] = 1 foundat['base'] = fileinput.filelineno() fileinput.close() - self.assertEqual(found['mod1'], 1, 'MOD1 does not appear once in RELEASE.local (found {0})'.format(found['mod1'])) - self.assertEqual(found['mod2'], 1, 'MOD2 does not appear once in RELEASE.local (found {0})'.format(found['mod2'])) - self.assertEqual(found['base'], 1, 'EPICS_BASE does not appear once in RELEASE.local (found {0})'.format(found['base'])) + self.assertEqual(found['mod1'], 1, + 'MOD1 does not appear once in RELEASE.local (found {0})'.format(found['mod1'])) + self.assertEqual(found['mod2'], 1, + 'MOD2 does not appear once in RELEASE.local (found {0})'.format(found['mod2'])) + self.assertEqual(found['base'], 1, + 'EPICS_BASE does not appear once in RELEASE.local (found {0})'.format(found['base'])) self.assertGreater(foundat['base'], foundat['mod2'], - 'EPICS_BASE (line {0}) appears before MOD2 (line {1})'.format(foundat['base'], foundat['mod2'])) + 'EPICS_BASE (line {0}) appears before MOD2 (line {1})' + .format(foundat['base'], foundat['mod2'])) self.assertGreater(foundat['mod2'], foundat['mod1'], 'MOD2 (line {0}) appears before MOD1 (line {1})'.format(foundat['mod2'], foundat['mod1'])) +class TestAddDependency(unittest.TestCase): + hash_3_15_6 = "ce7943fb44beb22b453ddcc0bda5398fadf72096" + location = os.path.join(do.cachedir, 'base-R3.15.6') + licensefile = os.path.join(location, 'LICENSE') + checked_file = os.path.join(location, 'checked_out') + release_file = os.path.join(location, 'configure', 'RELEASE') + def setUp(self): + os.environ['SETUP_PATH'] = '.:appveyor' + if os.path.exists(self.location): + shutil.rmtree(self.location) + do.clear_lists() + do.source_set('defaults') + + def test_MissingDependency(self): + do.add_dependency('BASE', 'R3.15.6') + self.assertTrue(os.path.exists(self.licensefile), 'Missing dependency was not checked out') + self.assertTrue(os.path.exists(self.checked_file), 'Checked-out commit marker was not written') + with open(self.checked_file, 'r') as bfile: + checked_out = bfile.read().strip() + bfile.close() + self.assertEqual(checked_out, self.hash_3_15_6, + 'Wrong commit of dependency checked out (expected=\"{0}\" found=\"{1}\")' + .format(self.hash_3_15_6, checked_out)) + self.assertFalse(find_in_file('include \$\(TOP\)/../RELEASE.local', self.release_file), + 'RELEASE in Base includes TOP/../RELEASE.local') + + def test_UpToDateDependency(self): + do.add_dependency('BASE', 'R3.15.6') + os.remove(self.licensefile) + do.add_dependency('BASE', 'R3.15.6') + self.assertFalse(os.path.exists(self.licensefile), 'Check out on top of existing up-to-date dependency') + + def test_OutdatedDependency(self): + do.add_dependency('BASE', 'R3.15.6') + os.remove(self.licensefile) + with open(self.checked_file, "w") as fout: + print('XXX not the right hash XXX', file=fout) + fout.close() + do.add_dependency('BASE', 'R3.15.6') + self.assertTrue(os.path.exists(self.licensefile), 'No check-out on top of out-of-date dependency') + with open(self.checked_file, 'r') as bfile: + checked_out = bfile.read().strip() + bfile.close() + self.assertEqual(checked_out, self.hash_3_15_6, + "Wrong commit of dependency checked out (expected='{0}' found='{1}')" + .format(self.hash_3_15_6, checked_out)) if __name__ == "__main__": # suite = unittest.TestLoader().loadTestsFromTestCase(TestSourceSet) # suite = unittest.TestLoader().loadTestsFromTestCase(TestUpdateReleaseLocal) +# suite = unittest.TestLoader().loadTestsFromTestCase(TestAddDependency) # unittest.TextTestRunner(verbosity=2).run(suite) unittest.main() diff --git a/appveyor/do.py b/appveyor/do.py index 2817315..bf8a822 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -4,9 +4,9 @@ from __future__ import print_function -import sys, os, fileinput +import sys, os, shutil, fileinput import logging -import subprocess as SP +import subprocess as sp import distutils.util logger = logging.getLogger(__name__) @@ -21,7 +21,9 @@ ANSI_CLEAR = "\033[0K" seen_setups = [] +modules_to_compile = [] setup = {} + if 'HomeDrive' in os.environ: cachedir = os.path.join(os.getenv('HomeDrive'), os.getenv('HomePath'), '.cache') elif 'HOME' in os.environ: @@ -32,21 +34,23 @@ # Used from unittests def clear_lists(): del seen_setups[:] + del modules_to_compile[:] setup.clear() # source_set(setup) # # Source a settings file (extension .set) found in the setup_dirs path # May be called recursively (from within a setup file) -def source_set(set): +def source_set(name): found = False + # allowed separators: colon or whitespace setup_dirs = os.getenv('SETUP_PATH', "").replace(':', ' ').split() if len(setup_dirs) == 0: raise NameError("{0}Search path for setup files (SETUP_PATH) is empty{1}".format(ANSI_RED,ANSI_RESET)) for set_dir in setup_dirs: - set_file = os.path.join(set_dir, set) + ".set" + set_file = os.path.join(set_dir, name) + ".set" if set_file in seen_setups: print("Ignoring already included setup file {0}".format(set_file)) @@ -100,16 +104,17 @@ def update_release_local(var, place): found = False logger.debug("Opening RELEASE.local for adding '%s'", updated_line) for line in fileinput.input(release_local, inplace=1): + outputline = line.strip() if 'EPICS_BASE=' in line: - logger.debug("Found EPICS_BASE line '%s', not writing it", base_line) base_line = line.strip() + logger.debug("Found EPICS_BASE line '%s', not writing it", base_line) continue elif '{0}='.format(var) in line: logger.debug("Found '%s=' line, replacing", var) found = True - line = updated_line + outputline = updated_line logger.debug("Writing line to RELEASE.local: '%s'", outputline) - print(line) + print(outputline) fileinput.close() fout = open(release_local,"a") if not found: @@ -120,6 +125,19 @@ def update_release_local(var, place): print(base_line, file=fout) fout.close() +def set_setup_from_env(dep): + for postf in ['_DIRNAME', '_REPONAME', '_REPOOWNER', '_REPOURL', + '_VARNAME', '_RECURSIVE', '_DEPTH', '_HOOK']: + if dep+postf in os.environ: + setup[dep+postf] = os.getenv(dep+postf) + +def call_git(args, **kws): + logger.debug("EXEC '%s' in %s", ' '.join(['git'] + args), os.getcwd()) + sys.stdout.flush() + exitcode = sp.call(['git'] + args, **kws) + logger.debug('EXEC DONE') + return exitcode + # add_dependency(dep, tag) # # Add a dependency to the cache area: @@ -135,7 +153,87 @@ def update_release_local(var, place): # - Add $dep_VARNAME line to the RELEASE.local file in the cache area (unless already there) # - Add full path to $modules_to_compile def add_dependency(dep, tag): - pass + curdir = os.getcwd() + set_setup_from_env(dep) + setup.setdefault(dep+"_DIRNAME", dep.lower()) + setup.setdefault(dep+"_REPONAME", dep.lower()) + setup.setdefault('REPOOWNER', 'epics-modules') + setup.setdefault(dep+"_REPOOWNER", setup['REPOOWNER']) + setup.setdefault(dep+"_REPOURL", 'https://github.com/{0}/{1}.git' + .format(setup[dep+'_REPOOWNER'], setup[dep+'_REPONAME'])) + setup.setdefault(dep+"_VARNAME", dep) + setup.setdefault(dep+"_RECURSIVE", 1) + setup.setdefault(dep+"_DEPTH", -1) + if setup[dep+'_RECURSIVE'] not in [0, 'no']: + recursearg = "--recursive" + else: + recursearg = '' + + # determine if dep points to a valid release or branch + if call_git(['ls-remote', '--quiet', '--exit-code', '--refs', setup[dep+'_REPOURL'], tag]): + raise RuntimeError("{0}{1} is neither a tag nor a branch name for {2} ({3}){4}" + .format(ANSI_RED, tag, dep, setup[dep+'_REPOURL'], ANSI_RESET)) + + dirname = setup[dep+'_DIRNAME']+'-{0}'.format(tag) + place = os.path.join(cachedir, dirname) + checked_file = os.path.join(place, "checked_out") + if os.path.isdir(place): + logger.debug('Dependency %s: directory %s exists, comparing checked-out commit', dep, place) + # check HEAD commit against the hash in marker file + if os.path.exists(checked_file): + with open(checked_file, 'r') as bfile: + checked_out = bfile.read().strip() + bfile.close() + else: + checked_out = 'never' + head = sp.check_output(['cd {0}; git log -n1 --pretty=format:%H'.format(place)], shell=True) + logger.debug('Found checked_out commit %s, git head is %s', checked_out, head) + if head != checked_out: + logger.debug('Dependency %s out of date - removing', dep) + shutil.rmtree(place) + else: + print('Found {0} of dependency {1} up-to-date in {2}'.format(tag, dep, place)) + + if not os.path.isdir(place): + if not os.path.isdir(cachedir): + os.makedirs(cachedir) + # clone dependency + os.chdir(cachedir) + deptharg = { + -1:['--depth', '5'], + 0:[], + }.get(setup[dep+'_DEPTH'], ['--depth', setup[dep+'_DEPTH']]) + print('Cloning {0} of dependency {1} into {2}' + .format(tag, dep, place)) + call_git(['clone', '--quiet'] + deptharg + [recursearg, '--branch', tag, setup[dep+'_REPOURL'], dirname]) + sp.check_call(['cd {0}; git log -n1'.format(place)], shell=True) + modules_to_compile.append(place) + + # force including RELEASE.local for non-base modules by overwriting their configure/RELEASE + if dep != 'BASE': + release = os.path.join(place, "configure", "RELEASE") + if os.path.exists(release): + fout = open(release, 'w') + print('-include $(TOP)/../RELEASE.local', file=fout) + fout.close() + + # run hook if defined + if dep+'_HOOK' in setup: + hook = os.path.join(place, setup[dep+'_HOOK']) + if os.path.exists(hook): + print('Running hook {0} in {1}'.format(setup[dep+'_HOOK'], place)) + os.chdir(place) + sp.check_call(hook, shell=True) + + # write checked out commit hash to marker file + head = sp.check_output(['cd {0}; git log -n1 --pretty=format:%H'.format(place)], shell=True) + logger.debug('Writing hash of checked-out dependency (%s) to marker file', head) + with open(checked_file, "w") as fout: + print(head, file=fout) + fout.close() + + update_release_local(setup[dep+"_VARNAME"], place) + os.chdir(curdir) def prepare(args): print(sys.version) @@ -145,10 +243,13 @@ def prepare(args): print('platform = ', distutils.util.get_platform()) print('{0}Loading setup files{1}'.format(ANSI_YELLOW, ANSI_RESET)) - source_set(default) + source_set('default') if 'SET' in os.environ: source_set(os.environ['SET']) + # we're working with tags (detached heads) a lot: suppress advice + call_git(['config', '--global', 'advice.detachedHead', 'false']) + print('Installing dependencies') def build(args): From 22d0feaa0555f061abf5335b29c3268db7fe7d32 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Wed, 19 Feb 2020 11:19:19 +0100 Subject: [PATCH 06/68] appveyor: enable debugging --- appveyor/do.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/appveyor/do.py b/appveyor/do.py index bf8a822..821a4ab 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -10,7 +10,7 @@ import distutils.util logger = logging.getLogger(__name__) -#logging.basicConfig(level=logging.DEBUG) +logging.basicConfig(level=logging.DEBUG) # Setup ANSI Colors ANSI_RED = "\033[31;1m" @@ -79,7 +79,7 @@ def source_set(name): if not found: raise NameError("{0}Setup file {1} does not exist in SETUP_PATH search path ({2}){3}" - .format(ANSI_RED,set_file,setup_dirs,ANSI_RESET)) + .format(ANSI_RED, name, setup_dirs, ANSI_RESET)) # update_release_local(var, place) # var name of the variable to set in RELEASE.local From 2dfa55420fa2e346527af098bc1ceadd8f715f35 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Wed, 19 Feb 2020 11:38:10 +0100 Subject: [PATCH 07/68] appveyor-test: always chdir into builddir --- appveyor-test.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/appveyor-test.py b/appveyor-test.py index a9643c5..2180eef 100644 --- a/appveyor-test.py +++ b/appveyor-test.py @@ -11,6 +11,8 @@ import re import unittest +builddir = os.getcwd() + def find_in_file(regex, filename): file = open (filename, "r") for line in file: @@ -39,6 +41,7 @@ def setUp(self): if 'BASE' in os.environ: del os.environ['BASE'] do.clear_lists() + os.chdir(builddir) def test_EmptySetupDirsPath(self): del os.environ['SETUP_PATH'] @@ -91,6 +94,7 @@ class TestUpdateReleaseLocal(unittest.TestCase): def setUp(self): if os.path.exists(self.release_local): os.remove(self.release_local) + os.chdir(builddir) def test_SetModule(self): do.update_release_local('MOD1', '/foo/bar') @@ -163,6 +167,7 @@ def setUp(self): if os.path.exists(self.location): shutil.rmtree(self.location) do.clear_lists() + os.chdir(builddir) do.source_set('defaults') def test_MissingDependency(self): From 28aeda558b49978ecd8ab38b4ff6bca4a38d6caf Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Wed, 19 Feb 2020 13:20:00 +0100 Subject: [PATCH 08/68] appveyor: use decode() on git hashes --- appveyor/do.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/appveyor/do.py b/appveyor/do.py index 821a4ab..d0a9064 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -186,7 +186,7 @@ def add_dependency(dep, tag): bfile.close() else: checked_out = 'never' - head = sp.check_output(['cd {0}; git log -n1 --pretty=format:%H'.format(place)], shell=True) + head = sp.check_output(['cd {0}; git log -n1 --pretty=format:%H'.format(place)], shell=True).decode() logger.debug('Found checked_out commit %s, git head is %s', checked_out, head) if head != checked_out: logger.debug('Dependency %s out of date - removing', dep) @@ -226,7 +226,7 @@ def add_dependency(dep, tag): sp.check_call(hook, shell=True) # write checked out commit hash to marker file - head = sp.check_output(['cd {0}; git log -n1 --pretty=format:%H'.format(place)], shell=True) + head = sp.check_output(['cd {0}; git log -n1 --pretty=format:%H'.format(place)], shell=True).decode() logger.debug('Writing hash of checked-out dependency (%s) to marker file', head) with open(checked_file, "w") as fout: print(head, file=fout) From 2847f78ab200c3f8c7be9c9c49d2b64f53f26d44 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Thu, 20 Feb 2020 09:15:35 +0100 Subject: [PATCH 09/68] appveyor: add error handler to fix shutil.rmtree on Windows --- appveyor-test.py | 2 +- appveyor/do.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/appveyor-test.py b/appveyor-test.py index 2180eef..7b3f3d8 100644 --- a/appveyor-test.py +++ b/appveyor-test.py @@ -165,7 +165,7 @@ class TestAddDependency(unittest.TestCase): def setUp(self): os.environ['SETUP_PATH'] = '.:appveyor' if os.path.exists(self.location): - shutil.rmtree(self.location) + shutil.rmtree(self.location, onerror=do.remove_readonly) do.clear_lists() os.chdir(builddir) do.source_set('defaults') diff --git a/appveyor/do.py b/appveyor/do.py index d0a9064..3b0ef55 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -4,7 +4,8 @@ from __future__ import print_function -import sys, os, shutil, fileinput +import sys, os, stat, shutil +import fileinput import logging import subprocess as sp import distutils.util @@ -37,6 +38,11 @@ def clear_lists(): del modules_to_compile[:] setup.clear() +# Error-handler to make shutil.rmtree delete read-only files on Windows +def remove_readonly(func, path, excinfo): + os.chmod(path, stat.S_IWRITE) + func(path) + # source_set(setup) # # Source a settings file (extension .set) found in the setup_dirs path @@ -190,7 +196,7 @@ def add_dependency(dep, tag): logger.debug('Found checked_out commit %s, git head is %s', checked_out, head) if head != checked_out: logger.debug('Dependency %s out of date - removing', dep) - shutil.rmtree(place) + shutil.rmtree(place, onerror=remove_readonly) else: print('Found {0} of dependency {1} up-to-date in {2}'.format(tag, dep, place)) From 9742c5f9c6e8d604943914a4960fd6c8579af848 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Tue, 17 Mar 2020 12:09:18 +0100 Subject: [PATCH 10/68] appveyor: use portable os.chdir() instead of "cd" --- appveyor/do.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/appveyor/do.py b/appveyor/do.py index 3b0ef55..ece2f7b 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -144,6 +144,16 @@ def call_git(args, **kws): logger.debug('EXEC DONE') return exitcode +def get_git_hash(place): + dir = os.getcwd() + os.chdir(place) + logger.debug("EXEC 'git log -n1 --pretty=format:%%H' in %s", place) + sys.stdout.flush() + head = sp.check_output(['git', 'log', '-n1', '--pretty=format:%H']).decode() + logger.debug('EXEC DONE') + os.chdir(dir) + return head + # add_dependency(dep, tag) # # Add a dependency to the cache area: @@ -183,6 +193,7 @@ def add_dependency(dep, tag): dirname = setup[dep+'_DIRNAME']+'-{0}'.format(tag) place = os.path.join(cachedir, dirname) checked_file = os.path.join(place, "checked_out") + if os.path.isdir(place): logger.debug('Dependency %s: directory %s exists, comparing checked-out commit', dep, place) # check HEAD commit against the hash in marker file @@ -192,7 +203,7 @@ def add_dependency(dep, tag): bfile.close() else: checked_out = 'never' - head = sp.check_output(['cd {0}; git log -n1 --pretty=format:%H'.format(place)], shell=True).decode() + head = get_git_hash(place) logger.debug('Found checked_out commit %s, git head is %s', checked_out, head) if head != checked_out: logger.debug('Dependency %s out of date - removing', dep) @@ -212,7 +223,8 @@ def add_dependency(dep, tag): print('Cloning {0} of dependency {1} into {2}' .format(tag, dep, place)) call_git(['clone', '--quiet'] + deptharg + [recursearg, '--branch', tag, setup[dep+'_REPOURL'], dirname]) - sp.check_call(['cd {0}; git log -n1'.format(place)], shell=True) + os.chdir(place) + sp.check_call(['git', 'log', '-n1']) modules_to_compile.append(place) # force including RELEASE.local for non-base modules by overwriting their configure/RELEASE @@ -232,7 +244,7 @@ def add_dependency(dep, tag): sp.check_call(hook, shell=True) # write checked out commit hash to marker file - head = sp.check_output(['cd {0}; git log -n1 --pretty=format:%H'.format(place)], shell=True).decode() + head = get_git_hash(place) logger.debug('Writing hash of checked-out dependency (%s) to marker file', head) with open(checked_file, "w") as fout: print(head, file=fout) From 4dcfbb2079fcf10583115a51baaf38c2bc453c13 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Tue, 17 Mar 2020 15:24:22 +0100 Subject: [PATCH 11/68] appveyor: add default repo URL test --- appveyor-test.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/appveyor-test.py b/appveyor-test.py index 7b3f3d8..34d023a 100644 --- a/appveyor-test.py +++ b/appveyor-test.py @@ -204,9 +204,40 @@ def test_OutdatedDependency(self): "Wrong commit of dependency checked out (expected='{0}' found='{1}')" .format(self.hash_3_15_6, checked_out)) +def repo_access(dep): + do.set_setup_from_env(dep) + do.setup.setdefault(dep + "_DIRNAME", dep.lower()) + do.setup.setdefault(dep + "_REPONAME", dep.lower()) + do.setup.setdefault('REPOOWNER', 'epics-modules') + do.setup.setdefault(dep + "_REPOOWNER", do.setup['REPOOWNER']) + do.setup.setdefault(dep + "_REPOURL", 'https://github.com/{0}/{1}.git' + .format(do.setup[dep + '_REPOOWNER'], do.setup[dep + '_REPONAME'])) + with open(os.devnull, 'w') as devnull: + return do.call_git(['ls-remote', '--quiet', '--heads', do.setup[dep + '_REPOURL']], + stdout=devnull, stderr=devnull) + +class TestDefaultModuleURLs(unittest.TestCase): + + modules = ['BASE', 'PVDATA', 'PVACCESS', 'NTYPES', + 'SNCSEQ', 'STREAM', 'ASYN', 'STD', + 'CALC', 'AUTOSAVE', 'BUSY', 'SSCAN', + 'IOCSTATS', 'MOTOR', 'IPAC', ] + + def setUp(self): + os.environ['SETUP_PATH'] = '.:appveyor' + do.clear_lists() + os.chdir(builddir) + do.source_set('defaults') + + def test_Repos(self): + for mod in self.modules: + self.assertEqual(repo_access(mod), 0, 'Defaults for {0} do not point to a valid git repository at {1}' + .format(mod, do.setup[mod + '_REPOURL'])) + if __name__ == "__main__": # suite = unittest.TestLoader().loadTestsFromTestCase(TestSourceSet) # suite = unittest.TestLoader().loadTestsFromTestCase(TestUpdateReleaseLocal) # suite = unittest.TestLoader().loadTestsFromTestCase(TestAddDependency) +# suite = unittest.TestLoader().loadTestsFromTestCase(TestDefaultModuleURLs) # unittest.TextTestRunner(verbosity=2).run(suite) unittest.main() From 249db7db22a9255093facd3b00fa67cb71e971e8 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Tue, 17 Mar 2020 17:36:12 +0100 Subject: [PATCH 12/68] appveyor: add cloning the dependency modules to 'prepare' action --- .appveyor.yml | 8 ++++++++ appveyor-test.py | 14 +++++++------- appveyor/do.py | 31 ++++++++++++++++++++++++------- 3 files changed, 39 insertions(+), 14 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index b336a73..c9ba9a6 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -29,6 +29,7 @@ skip_commits: - '**/*.html' - '**/*.md' + #---------------------------------# # build matrix configuration # #---------------------------------# @@ -42,7 +43,14 @@ configuration: # Environment variables: compiler toolchain environment: + # common variables + SETUP_PATH: .:.ci + SET: test01 + matrix: + - CC: vs2019 + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 + SET: test00 - CC: mingw APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 - CC: vs2019 diff --git a/appveyor-test.py b/appveyor-test.py index 34d023a..da45485 100644 --- a/appveyor-test.py +++ b/appveyor-test.py @@ -2,7 +2,7 @@ """Module ci-scripts AppVeyor unit tests """ -# SET=test00 in .appveyor.yml runs the tests in this script +# SET=test00 in the environment (.appveyor.yml) runs the tests in this script # all other jobs are started as compile jobs from __future__ import print_function @@ -235,9 +235,9 @@ def test_Repos(self): .format(mod, do.setup[mod + '_REPOURL'])) if __name__ == "__main__": -# suite = unittest.TestLoader().loadTestsFromTestCase(TestSourceSet) -# suite = unittest.TestLoader().loadTestsFromTestCase(TestUpdateReleaseLocal) -# suite = unittest.TestLoader().loadTestsFromTestCase(TestAddDependency) -# suite = unittest.TestLoader().loadTestsFromTestCase(TestDefaultModuleURLs) -# unittest.TextTestRunner(verbosity=2).run(suite) - unittest.main() + if 'SET' in os.environ and os.environ['SET'] == "test00": + unittest.main() + else: + do.actions['prepare']() + do.actions['build']() + do.actions['test']() diff --git a/appveyor/do.py b/appveyor/do.py index ece2f7b..dfe741d 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -253,7 +253,7 @@ def add_dependency(dep, tag): update_release_local(setup[dep+"_VARNAME"], place) os.chdir(curdir) -def prepare(args): +def prepare(): print(sys.version) print('PYTHONPATH') for dname in sys.path: @@ -261,19 +261,36 @@ def prepare(args): print('platform = ', distutils.util.get_platform()) print('{0}Loading setup files{1}'.format(ANSI_YELLOW, ANSI_RESET)) - source_set('default') + source_set('defaults') if 'SET' in os.environ: source_set(os.environ['SET']) # we're working with tags (detached heads) a lot: suppress advice call_git(['config', '--global', 'advice.detachedHead', 'false']) - print('Installing dependencies') - -def build(args): + print('{0}Checking/cloning dependencies{1}'.format(ANSI_YELLOW, ANSI_RESET)) + + add_modules = '' + if 'ADD_MODULES' in os.environ: + add_modules = os.environ['ADD_MODULES'] + modules = '' + if 'MODULES' in os.environ: + modules = os.environ['MODULES'] + modlist = 'BASE {0} {1}'.format(add_modules, modules).upper().split() + for mod in modlist: + if not setup[mod].strip(): + setup[mod] = 'master' + logger.debug('Adding dependency %s with tag %s', mod, setup[mod]) + add_dependency(mod, setup[mod]) + + if os.path.isdir('configure'): + release_local = os.path.join(cachedir, 'RELEASE.local') + shutil.copy(release_local, 'configure') + +def build(): print('Building the module') -def test(args): +def test(): print('Running the tests') actions = { @@ -287,4 +304,4 @@ def test(args): while len(args)>0: name = args.pop(0) print('IN', name, 'with', args) - actions[name](args) + actions[name]() From 5d17fdf98d49d4ca4ff6481bf529c6124b15e474 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Wed, 18 Mar 2020 16:57:46 +0100 Subject: [PATCH 13/68] appveyor: configure EPICS build; install make and Perl --- appveyor/do.py | 64 +++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 56 insertions(+), 8 deletions(-) diff --git a/appveyor/do.py b/appveyor/do.py index dfe741d..e120d33 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -7,6 +7,7 @@ import sys, os, stat, shutil import fileinput import logging +import re import subprocess as sp import distutils.util @@ -24,19 +25,26 @@ seen_setups = [] modules_to_compile = [] setup = {} +place = {} if 'HomeDrive' in os.environ: cachedir = os.path.join(os.getenv('HomeDrive'), os.getenv('HomePath'), '.cache') + toolsdir = os.path.join(os.getenv('HomeDrive'), os.getenv('HomePath'), '.tools') elif 'HOME' in os.environ: cachedir = os.path.join(os.getenv('HOME'), '.cache') + toolsdir = os.path.join(os.getenv('HOME'), '.tools') else: cachedir = os.path.join('.', '.cache') + toolsdir = os.path.join('.', '.tools') + +zip7 = 'C:\\Program Files\\7-Zip\\7z' # Used from unittests def clear_lists(): del seen_setups[:] del modules_to_compile[:] setup.clear() + place.clear() # Error-handler to make shutil.rmtree delete read-only files on Windows def remove_readonly(func, path, excinfo): @@ -87,16 +95,18 @@ def source_set(name): raise NameError("{0}Setup file {1} does not exist in SETUP_PATH search path ({2}){3}" .format(ANSI_RED, name, setup_dirs, ANSI_RESET)) -# update_release_local(var, place) -# var name of the variable to set in RELEASE.local -# place place (absolute path) of where variable should point to +# update_release_local(var, location) +# var name of the variable to set in RELEASE.local +# location location (absolute path) of where variable should point to # # Manipulate RELEASE.local in the cache location: -# - replace "$var=$place" line if it exists and has changed -# - otherwise add "$var=$place" line and possibly move EPICS_BASE=... line to the end -def update_release_local(var, place): +# - replace "$var=$location" line if it exists and has changed +# - otherwise add "$var=$location" line and possibly move EPICS_BASE=... line to the end +# Set place[var] = location +def update_release_local(var, location): release_local = os.path.join(cachedir, 'RELEASE.local') - updated_line = '{0}={1}'.format(var, place) + updated_line = '{0}={1}'.format(var, location) + place[var] = location if not os.path.exists(release_local): logger.debug('RELEASE.local does not exist, creating it') @@ -287,8 +297,46 @@ def prepare(): release_local = os.path.join(cachedir, 'RELEASE.local') shutil.copy(release_local, 'configure') + print('{0}Setting up EPICS build system{1}'.format(ANSI_YELLOW, ANSI_RESET)) + + with open(os.path.join(place['EPICS_BASE'], 'configure', 'CONFIG_SITE'), 'a') as config_site: + if re.search('static', os.environ['CONFIGURATION']): + config_site.write('SHARED_LIBRARIES=NO') + config_site.write('STATIC_BUILD=YES') + linktype = 'static' + else: + linktype = 'dynamic (DLL)' + if re.search('debug', os.environ['CONFIGURATION']): + config_site.write('HOST_OPT=NO') + optitype = 'debug' + else: + optitype = 'optimized' + + print('EPICS Base set up for {0} build with {1} linking'.format(optitype, linktype)) + + if not os.path.isdir(toolsdir): + os.makedirs(toolsdir) + + print('Installing Make 4.2.1 from ANL web site') + sys.stdout.flush() + os.chdir(toolsdir) + sp.check_call(['curl', '-fsS', '--retry', '3', '-o', 'make-4.2.1.zip', + 'https://epics.anl.gov/download/tools/make-4.2.1-win64.zip']) + sp.check_call([zip7, 'e', 'make-4.2.1.zip']) + + perlver = '5.30.0.1' + if os.environ['CC'] == 'vs2019': + print('Installing Strawberry Perl {0}'.format(perlver)) + sys.stdout.flush() + os.chdir(toolsdir) + sp.check_call(['curl', '-fsS', '--retry', '3', '-o', 'perl-{0}.zip'.format(perlver), + 'http://strawberryperl.com/download/{0}/strawberry-perl-{0}-64bit.zip'.format(perlver)]) + sp.check_call([zip7, 'x', 'perl-{0}.zip'.format(perlver), '-ostrawberry']) + os.chdir('strawberry') + sp.check_call('relocation.pl.bat', shell=True) + def build(): - print('Building the module') + print('{0}Building the module{1}'.format(ANSI_YELLOW, ANSI_RESET)) def test(): print('Running the tests') From e2399dc7f3544c4e583e0b7c3ca798e9fa4b02bc Mon Sep 17 00:00:00 2001 From: Michael Davidsaver Date: Fri, 20 Mar 2020 21:05:42 -0700 Subject: [PATCH 14/68] appveyor: avoid chdir, use 'cwd' key instead --- appveyor/do.py | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/appveyor/do.py b/appveyor/do.py index e120d33..324f7dc 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -155,13 +155,10 @@ def call_git(args, **kws): return exitcode def get_git_hash(place): - dir = os.getcwd() - os.chdir(place) logger.debug("EXEC 'git log -n1 --pretty=format:%%H' in %s", place) sys.stdout.flush() - head = sp.check_output(['git', 'log', '-n1', '--pretty=format:%H']).decode() + head = sp.check_output(['git', 'log', '-n1', '--pretty=format:%H'], cwd=place).decode() logger.debug('EXEC DONE') - os.chdir(dir) return head # add_dependency(dep, tag) @@ -179,7 +176,6 @@ def get_git_hash(place): # - Add $dep_VARNAME line to the RELEASE.local file in the cache area (unless already there) # - Add full path to $modules_to_compile def add_dependency(dep, tag): - curdir = os.getcwd() set_setup_from_env(dep) setup.setdefault(dep+"_DIRNAME", dep.lower()) setup.setdefault(dep+"_REPONAME", dep.lower()) @@ -225,15 +221,14 @@ def add_dependency(dep, tag): if not os.path.isdir(cachedir): os.makedirs(cachedir) # clone dependency - os.chdir(cachedir) deptharg = { -1:['--depth', '5'], 0:[], }.get(setup[dep+'_DEPTH'], ['--depth', setup[dep+'_DEPTH']]) print('Cloning {0} of dependency {1} into {2}' .format(tag, dep, place)) - call_git(['clone', '--quiet'] + deptharg + [recursearg, '--branch', tag, setup[dep+'_REPOURL'], dirname]) - os.chdir(place) + call_git(['clone', '--quiet'] + deptharg + [recursearg, '--branch', tag, setup[dep+'_REPOURL'], dirname], cwd=cachedir) + sp.check_call(['git', 'log', '-n1']) modules_to_compile.append(place) @@ -250,8 +245,8 @@ def add_dependency(dep, tag): hook = os.path.join(place, setup[dep+'_HOOK']) if os.path.exists(hook): print('Running hook {0} in {1}'.format(setup[dep+'_HOOK'], place)) - os.chdir(place) - sp.check_call(hook, shell=True) + + sp.check_call(hook, shell=True, cwd=place) # write checked out commit hash to marker file head = get_git_hash(place) @@ -261,7 +256,6 @@ def add_dependency(dep, tag): fout.close() update_release_local(setup[dep+"_VARNAME"], place) - os.chdir(curdir) def prepare(): print(sys.version) @@ -319,25 +313,30 @@ def prepare(): print('Installing Make 4.2.1 from ANL web site') sys.stdout.flush() - os.chdir(toolsdir) + sp.check_call(['curl', '-fsS', '--retry', '3', '-o', 'make-4.2.1.zip', - 'https://epics.anl.gov/download/tools/make-4.2.1-win64.zip']) - sp.check_call([zip7, 'e', 'make-4.2.1.zip']) + 'https://epics.anl.gov/download/tools/make-4.2.1-win64.zip'], + cwd=toolsdir) + sp.check_call([zip7, 'e', 'make-4.2.1.zip'], cwd=toolsdir) perlver = '5.30.0.1' if os.environ['CC'] == 'vs2019': print('Installing Strawberry Perl {0}'.format(perlver)) sys.stdout.flush() - os.chdir(toolsdir) + sp.check_call(['curl', '-fsS', '--retry', '3', '-o', 'perl-{0}.zip'.format(perlver), - 'http://strawberryperl.com/download/{0}/strawberry-perl-{0}-64bit.zip'.format(perlver)]) - sp.check_call([zip7, 'x', 'perl-{0}.zip'.format(perlver), '-ostrawberry']) - os.chdir('strawberry') - sp.check_call('relocation.pl.bat', shell=True) + 'http://strawberryperl.com/download/{0}/strawberry-perl-{0}-64bit.zip'.format(perlver)], + cwd=toolsdir) + sp.check_call([zip7, 'x', 'perl-{0}.zip'.format(perlver), '-ostrawberry'], cwd=toolsdir) + + sp.check_call('relocation.pl.bat', shell=True, + cwd=os.path.join(toolsdir, 'strawberry')) def build(): print('{0}Building the module{1}'.format(ANSI_YELLOW, ANSI_RESET)) + + def test(): print('Running the tests') From c3918cdbaa5502a270bf06dd857efdfb966a935c Mon Sep 17 00:00:00 2001 From: Michael Davidsaver Date: Fri, 20 Mar 2020 21:45:12 -0700 Subject: [PATCH 15/68] appveyor: add printing host_info (python settings, VS versions) --- appveyor-test.py | 1 + appveyor/do.py | 21 ++++++++++++++++----- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/appveyor-test.py b/appveyor-test.py index da45485..6b84316 100644 --- a/appveyor-test.py +++ b/appveyor-test.py @@ -235,6 +235,7 @@ def test_Repos(self): .format(mod, do.setup[mod + '_REPOURL'])) if __name__ == "__main__": + do.host_info() if 'SET' in os.environ and os.environ['SET'] == "test00": unittest.main() else: diff --git a/appveyor/do.py b/appveyor/do.py index 324f7dc..d633a4f 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -39,6 +39,21 @@ zip7 = 'C:\\Program Files\\7-Zip\\7z' +def host_info(): + print(sys.version) + print('PYTHONPATH') + for dname in sys.path: + print(' ', dname) + print('platform = ', distutils.util.get_platform()) + + print('Listing available VS versions') + from fnmatch import fnmatch + for base in (r'C:\Program Files (x86)', r'C:\Program Files'): + for root, dirs, files in os.walk(base): + for fname in files: + if fnmatch(fname, 'vcvars*.bat'): + print('Found', os.path.join(root, fname)) + # Used from unittests def clear_lists(): del seen_setups[:] @@ -258,11 +273,7 @@ def add_dependency(dep, tag): update_release_local(setup[dep+"_VARNAME"], place) def prepare(): - print(sys.version) - print('PYTHONPATH') - for dname in sys.path: - print(' ', dname) - print('platform = ', distutils.util.get_platform()) + host_info() print('{0}Loading setup files{1}'.format(ANSI_YELLOW, ANSI_RESET)) source_set('defaults') From 00f003afa55baf54d2094483b5d8e6b484eae72c Mon Sep 17 00:00:00 2001 From: Michael Davidsaver Date: Fri, 20 Mar 2020 23:31:37 -0700 Subject: [PATCH 16/68] appveyor: add with_vcvars to read VS environment settings - writes and calls a "trampoline" batch that calls the appropriate "vcvarsall" script, then calls back into python --- appveyor-test.py | 13 ++++++++++- appveyor/do.py | 59 ++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 66 insertions(+), 6 deletions(-) diff --git a/appveyor-test.py b/appveyor-test.py index 6b84316..4b9c0d4 100644 --- a/appveyor-test.py +++ b/appveyor-test.py @@ -234,10 +234,21 @@ def test_Repos(self): self.assertEqual(repo_access(mod), 0, 'Defaults for {0} do not point to a valid git repository at {1}' .format(mod, do.setup[mod + '_REPOURL'])) +class TestVCVars(unittest.TestCase): + def test_vcvars(self): + if os.environ['CC'] in ('mingw',): + raise unittest.SkipTest() + + do.with_vcvars('env') + if __name__ == "__main__": do.host_info() if 'SET' in os.environ and os.environ['SET'] == "test00": - unittest.main() + if sys.argv[1:]==['env']: + # testing with_vcvars + [print(K,'=',V) for K, V in os.environ.items()] + else: + unittest.main() else: do.actions['prepare']() do.actions['build']() diff --git a/appveyor/do.py b/appveyor/do.py index d633a4f..5ae0afc 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -347,19 +347,68 @@ def build(): print('{0}Building the module{1}'.format(ANSI_YELLOW, ANSI_RESET)) - def test(): print('Running the tests') +def with_vcvars(cmd): + '''re-exec main script with a (hopefully different) command + ''' + CC = os.environ['CC'] + + # cf. https://docs.microsoft.com/en-us/cpp/build/building-on-the-command-line + + info = { + 'python':sys.executable, + 'self':sys.argv[0], + 'cmd':cmd, + } + + info['arch'] = { + 'x86':'x86', # 'amd64_x86' ?? + 'x64':'amd64', + }[os.environ['PLATFORM']] # 'x86' or 'x64' + + info['vcvars'] = { + # https://en.wikipedia.org/wiki/Microsoft_Visual_Studio#History + 'vs2019':r'C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvarsall.bat', + 'vs2017':r'C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvarsall.bat', + 'vs2015':r'C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat', + 'vs2013':r'C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat', + 'vs2012':r'C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\vcvarsall.bat', + 'vs2010':r'C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat', + 'vs2008':r'C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\vcvarsall.bat', + }[CC] + + script=''' +call "{vcvars}" {arch} + +"{python}" "{self}" {cmd} +'''.format(**info) + + print('vcvars-trampoline.bat') + print(script) + + with open('vcvars-trampoline.bat', 'w') as F: + F.write(script) + + sys.stdout.flush() + sp.check_call('vcvars-trampoline.bat', shell=True) + actions = { 'prepare': prepare, 'build': build, 'test': test, + '_vcvars':lambda:None, } if __name__=='__main__': args = sys.argv[1:] - while len(args)>0: - name = args.pop(0) - print('IN', name, 'with', args) - actions[name]() + if args[0]!='_vcvars' and os.environ['CC']!='mingw': + # re-exec with MSVC in PATH + with_vcvars(' '.join(['_vcvars']+args)) + + else: + while len(args)>0: + name = args.pop(0) + print('IN', name, 'with', args) + actions[name]() From 6071fdf198908a51ddb171811005a59a73c59dbe Mon Sep 17 00:00:00 2001 From: Michael Davidsaver Date: Sat, 21 Mar 2020 08:21:21 -0700 Subject: [PATCH 17/68] appveyor: minor fixes - using 'place' as both a local and a global is confusing - use "with open()" instead of "open() / close()" --- appveyor/do.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/appveyor/do.py b/appveyor/do.py index 5ae0afc..bd3a675 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -25,7 +25,7 @@ seen_setups = [] modules_to_compile = [] setup = {} -place = {} +places = {} if 'HomeDrive' in os.environ: cachedir = os.path.join(os.getenv('HomeDrive'), os.getenv('HomePath'), '.cache') @@ -59,7 +59,7 @@ def clear_lists(): del seen_setups[:] del modules_to_compile[:] setup.clear() - place.clear() + places.clear() # Error-handler to make shutil.rmtree delete read-only files on Windows def remove_readonly(func, path, excinfo): @@ -117,11 +117,11 @@ def source_set(name): # Manipulate RELEASE.local in the cache location: # - replace "$var=$location" line if it exists and has changed # - otherwise add "$var=$location" line and possibly move EPICS_BASE=... line to the end -# Set place[var] = location +# Set places[var] = location def update_release_local(var, location): release_local = os.path.join(cachedir, 'RELEASE.local') updated_line = '{0}={1}'.format(var, location) - place[var] = location + places[var] = location if not os.path.exists(release_local): logger.debug('RELEASE.local does not exist, creating it') @@ -251,9 +251,8 @@ def add_dependency(dep, tag): if dep != 'BASE': release = os.path.join(place, "configure", "RELEASE") if os.path.exists(release): - fout = open(release, 'w') - print('-include $(TOP)/../RELEASE.local', file=fout) - fout.close() + with open(release, 'w') as fout: + print('-include $(TOP)/../RELEASE.local', file=fout) # run hook if defined if dep+'_HOOK' in setup: @@ -304,7 +303,7 @@ def prepare(): print('{0}Setting up EPICS build system{1}'.format(ANSI_YELLOW, ANSI_RESET)) - with open(os.path.join(place['EPICS_BASE'], 'configure', 'CONFIG_SITE'), 'a') as config_site: + with open(os.path.join(places['EPICS_BASE'], 'configure', 'CONFIG_SITE'), 'a') as config_site: if re.search('static', os.environ['CONFIGURATION']): config_site.write('SHARED_LIBRARIES=NO') config_site.write('STATIC_BUILD=YES') From e47e35bae4487c0d27b3c36cb7b0eab567936959 Mon Sep 17 00:00:00 2001 From: Michael Davidsaver Date: Sat, 21 Mar 2020 08:29:01 -0700 Subject: [PATCH 18/68] appveyor: add do_exec() action; 'make' dependencies --- appveyor-test.py | 13 ++++++++----- appveyor/do.py | 41 ++++++++++++++++++++++++++++------------- 2 files changed, 36 insertions(+), 18 deletions(-) diff --git a/appveyor-test.py b/appveyor-test.py index 4b9c0d4..1a9155c 100644 --- a/appveyor-test.py +++ b/appveyor-test.py @@ -171,7 +171,8 @@ def setUp(self): do.source_set('defaults') def test_MissingDependency(self): - do.add_dependency('BASE', 'R3.15.6') + do.setup['BASE'] = 'R3.15.6' + do.add_dependency('BASE') self.assertTrue(os.path.exists(self.licensefile), 'Missing dependency was not checked out') self.assertTrue(os.path.exists(self.checked_file), 'Checked-out commit marker was not written') with open(self.checked_file, 'r') as bfile: @@ -184,18 +185,20 @@ def test_MissingDependency(self): 'RELEASE in Base includes TOP/../RELEASE.local') def test_UpToDateDependency(self): - do.add_dependency('BASE', 'R3.15.6') + do.setup['BASE'] = 'R3.15.6' + do.add_dependency('BASE') os.remove(self.licensefile) - do.add_dependency('BASE', 'R3.15.6') + do.add_dependency('BASE') self.assertFalse(os.path.exists(self.licensefile), 'Check out on top of existing up-to-date dependency') def test_OutdatedDependency(self): - do.add_dependency('BASE', 'R3.15.6') + do.setup['BASE'] = 'R3.15.6' + do.add_dependency('BASE') os.remove(self.licensefile) with open(self.checked_file, "w") as fout: print('XXX not the right hash XXX', file=fout) fout.close() - do.add_dependency('BASE', 'R3.15.6') + do.add_dependency('BASE') self.assertTrue(os.path.exists(self.licensefile), 'No check-out on top of out-of-date dependency') with open(self.checked_file, 'r') as bfile: checked_out = bfile.read().strip() diff --git a/appveyor/do.py b/appveyor/do.py index bd3a675..437c2c9 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -37,6 +37,9 @@ cachedir = os.path.join('.', '.cache') toolsdir = os.path.join('.', '.tools') +# ensure our 'make' found first +os.environ['PATH'] = os.pathsep.join([toolsdir, os.environ['PATH']]) + zip7 = 'C:\\Program Files\\7-Zip\\7z' def host_info(): @@ -190,8 +193,9 @@ def get_git_hash(place): # $dep_RECURSIVE = 1/YES (0/NO to for a flat clone) # - Add $dep_VARNAME line to the RELEASE.local file in the cache area (unless already there) # - Add full path to $modules_to_compile -def add_dependency(dep, tag): +def add_dependency(dep): set_setup_from_env(dep) + setup.setdefault(dep, 'master') setup.setdefault(dep+"_DIRNAME", dep.lower()) setup.setdefault(dep+"_REPONAME", dep.lower()) setup.setdefault('REPOOWNER', 'epics-modules') @@ -206,6 +210,10 @@ def add_dependency(dep, tag): else: recursearg = '' + tag = setup[dep] + + logger.debug('Adding dependency %s with tag %s', dep, setup[dep]) + # determine if dep points to a valid release or branch if call_git(['ls-remote', '--quiet', '--exit-code', '--refs', setup[dep+'_REPOURL'], tag]): raise RuntimeError("{0}{1} is neither a tag nor a branch name for {2} ({3}){4}" @@ -271,7 +279,7 @@ def add_dependency(dep, tag): update_release_local(setup[dep+"_VARNAME"], place) -def prepare(): +def prepare(*args): host_info() print('{0}Loading setup files{1}'.format(ANSI_YELLOW, ANSI_RESET)) @@ -291,11 +299,7 @@ def prepare(): if 'MODULES' in os.environ: modules = os.environ['MODULES'] modlist = 'BASE {0} {1}'.format(add_modules, modules).upper().split() - for mod in modlist: - if not setup[mod].strip(): - setup[mod] = 'master' - logger.debug('Adding dependency %s with tag %s', mod, setup[mod]) - add_dependency(mod, setup[mod]) + [add_dependency(mod) for mod in modlist] if os.path.isdir('configure'): release_local = os.path.join(cachedir, 'RELEASE.local') @@ -342,13 +346,24 @@ def prepare(): sp.check_call('relocation.pl.bat', shell=True, cwd=os.path.join(toolsdir, 'strawberry')) -def build(): + for mod in modlist: + place = places[setup[mod+"_VARNAME"]] + print('Building '+place) + sp.check_call('make', shell=True, cwd=place) + +def build(*args): print('{0}Building the module{1}'.format(ANSI_YELLOW, ANSI_RESET)) -def test(): +def test(*args): print('Running the tests') +def doExec(*args): + 'exec user command with vcvars' + print('Execute command {}'.format(args)) + + sp.check_call(' '.join(args), shell=True) + def with_vcvars(cmd): '''re-exec main script with a (hopefully different) command ''' @@ -397,6 +412,7 @@ def with_vcvars(cmd): 'prepare': prepare, 'build': build, 'test': test, + 'exec': doExec, '_vcvars':lambda:None, } @@ -407,7 +423,6 @@ def with_vcvars(cmd): with_vcvars(' '.join(['_vcvars']+args)) else: - while len(args)>0: - name = args.pop(0) - print('IN', name, 'with', args) - actions[name]() + name = args.pop(0) + print('IN', name, 'with', args) + actions[name](*args) From b53468e50edf23bad87d141da21cf3842585ad63 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Mon, 23 Mar 2020 10:37:28 +0100 Subject: [PATCH 19/68] appveyor: make modlist a function --- appveyor/do.py | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/appveyor/do.py b/appveyor/do.py index 437c2c9..52f8c88 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -40,6 +40,14 @@ # ensure our 'make' found first os.environ['PATH'] = os.pathsep.join([toolsdir, os.environ['PATH']]) + +def modlist(): + add_modules = os.environ.get('ADD_MODULES', '').upper().split() + modules = os.environ.get('MODULES', '').upper().split() + ret = ['BASE'] + add_modules + modules + logger.debug('Effective module list: %s', ret) + return ret + zip7 = 'C:\\Program Files\\7-Zip\\7z' def host_info(): @@ -74,8 +82,6 @@ def remove_readonly(func, path, excinfo): # Source a settings file (extension .set) found in the setup_dirs path # May be called recursively (from within a setup file) def source_set(name): - found = False - # allowed separators: colon or whitespace setup_dirs = os.getenv('SETUP_PATH', "").replace(':', ' ').split() if len(setup_dirs) == 0: @@ -106,10 +112,8 @@ def source_set(name): if not setup[assign[0]].strip(): logger.debug('Doing assignment: %s = %s', assign[0], assign[1]) setup[assign[0]] = assign[1] - found = True break - - if not found: + else: raise NameError("{0}Setup file {1} does not exist in SETUP_PATH search path ({2}){3}" .format(ANSI_RED, name, setup_dirs, ANSI_RESET)) @@ -160,10 +164,11 @@ def update_release_local(var, location): fout.close() def set_setup_from_env(dep): - for postf in ['_DIRNAME', '_REPONAME', '_REPOOWNER', '_REPOURL', + for postf in ['', '_DIRNAME', '_REPONAME', '_REPOOWNER', '_REPOURL', '_VARNAME', '_RECURSIVE', '_DEPTH', '_HOOK']: if dep+postf in os.environ: - setup[dep+postf] = os.getenv(dep+postf) + setup[dep+postf] = os.environ[dep+postf] + logger.debug('ENV assignment: %s = %s', dep+postf, setup[dep+postf]) def call_git(args, **kws): logger.debug("EXEC '%s' in %s", ' '.join(['git'] + args), os.getcwd()) @@ -292,14 +297,7 @@ def prepare(*args): print('{0}Checking/cloning dependencies{1}'.format(ANSI_YELLOW, ANSI_RESET)) - add_modules = '' - if 'ADD_MODULES' in os.environ: - add_modules = os.environ['ADD_MODULES'] - modules = '' - if 'MODULES' in os.environ: - modules = os.environ['MODULES'] - modlist = 'BASE {0} {1}'.format(add_modules, modules).upper().split() - [add_dependency(mod) for mod in modlist] + [add_dependency(mod) for mod in modlist()] if os.path.isdir('configure'): release_local = os.path.join(cachedir, 'RELEASE.local') @@ -346,7 +344,7 @@ def prepare(*args): sp.check_call('relocation.pl.bat', shell=True, cwd=os.path.join(toolsdir, 'strawberry')) - for mod in modlist: + for mod in modlist(): place = places[setup[mod+"_VARNAME"]] print('Building '+place) sp.check_call('make', shell=True, cwd=place) From d052350738f8fc8b9ddccfd65f353badf9c41c54 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Mon, 23 Mar 2020 10:52:14 +0100 Subject: [PATCH 20/68] appveyor: add complete_setup() and do a complete log --- appveyor-test.py | 1 + appveyor/do.py | 31 ++++++++++++++++++++----------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/appveyor-test.py b/appveyor-test.py index 1a9155c..a270c0f 100644 --- a/appveyor-test.py +++ b/appveyor-test.py @@ -169,6 +169,7 @@ def setUp(self): do.clear_lists() os.chdir(builddir) do.source_set('defaults') + do.complete_setup('BASE') def test_MissingDependency(self): do.setup['BASE'] = 'R3.15.6' diff --git a/appveyor/do.py b/appveyor/do.py index 52f8c88..6ef4793 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -184,6 +184,19 @@ def get_git_hash(place): logger.debug('EXEC DONE') return head +def complete_setup(dep): + set_setup_from_env(dep) + setup.setdefault(dep, 'master') + setup.setdefault(dep+"_DIRNAME", dep.lower()) + setup.setdefault(dep+"_REPONAME", dep.lower()) + setup.setdefault('REPOOWNER', 'epics-modules') + setup.setdefault(dep+"_REPOOWNER", setup['REPOOWNER']) + setup.setdefault(dep+"_REPOURL", 'https://github.com/{0}/{1}.git' + .format(setup[dep+'_REPOOWNER'], setup[dep+'_REPONAME'])) + setup.setdefault(dep+"_VARNAME", dep) + setup.setdefault(dep+"_RECURSIVE", 1) + setup.setdefault(dep+"_DEPTH", -1) + # add_dependency(dep, tag) # # Add a dependency to the cache area: @@ -199,17 +212,6 @@ def get_git_hash(place): # - Add $dep_VARNAME line to the RELEASE.local file in the cache area (unless already there) # - Add full path to $modules_to_compile def add_dependency(dep): - set_setup_from_env(dep) - setup.setdefault(dep, 'master') - setup.setdefault(dep+"_DIRNAME", dep.lower()) - setup.setdefault(dep+"_REPONAME", dep.lower()) - setup.setdefault('REPOOWNER', 'epics-modules') - setup.setdefault(dep+"_REPOOWNER", setup['REPOOWNER']) - setup.setdefault(dep+"_REPOURL", 'https://github.com/{0}/{1}.git' - .format(setup[dep+'_REPOOWNER'], setup[dep+'_REPONAME'])) - setup.setdefault(dep+"_VARNAME", dep) - setup.setdefault(dep+"_RECURSIVE", 1) - setup.setdefault(dep+"_DEPTH", -1) if setup[dep+'_RECURSIVE'] not in [0, 'no']: recursearg = "--recursive" else: @@ -292,6 +294,13 @@ def prepare(*args): if 'SET' in os.environ: source_set(os.environ['SET']) + [complete_setup(mod) for mod in modlist()] + + logger.debug('Loaded setup') + kvs = list(setup.items()) + kvs.sort() + [logger.debug(' %s = "%s"', *kv) for kv in kvs] + # we're working with tags (detached heads) a lot: suppress advice call_git(['config', '--global', 'advice.detachedHead', 'false']) From 08cdff949550428eeb845f712f500744f0d13027 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Mon, 23 Mar 2020 11:00:30 +0100 Subject: [PATCH 21/68] appveyor: don't run host_info() twice for compile tests --- appveyor-test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor-test.py b/appveyor-test.py index a270c0f..33d8ba2 100644 --- a/appveyor-test.py +++ b/appveyor-test.py @@ -246,8 +246,8 @@ def test_vcvars(self): do.with_vcvars('env') if __name__ == "__main__": - do.host_info() if 'SET' in os.environ and os.environ['SET'] == "test00": + do.host_info() if sys.argv[1:]==['env']: # testing with_vcvars [print(K,'=',V) for K, V in os.environ.items()] From d5c419bc8e300cf5428c5c8f9baa9f2e130c4811 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Mon, 23 Mar 2020 11:38:26 +0100 Subject: [PATCH 22/68] appveyor: don't run TestVCVars unless on Windows --- appveyor-test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/appveyor-test.py b/appveyor-test.py index 33d8ba2..61f1e62 100644 --- a/appveyor-test.py +++ b/appveyor-test.py @@ -8,6 +8,7 @@ from __future__ import print_function import sys, os, shutil, fileinput +import distutils.util import re import unittest @@ -240,7 +241,8 @@ def test_Repos(self): class TestVCVars(unittest.TestCase): def test_vcvars(self): - if os.environ['CC'] in ('mingw',): + if ('CC' in os.environ and os.environ['CC'] in ('mingw',)) \ + or distutils.util.get_platform() != "win32": raise unittest.SkipTest() do.with_vcvars('env') From 85f570ac09f5b0132e6a3b336333a31e9d81bfa3 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Mon, 23 Mar 2020 12:27:05 +0100 Subject: [PATCH 23/68] appveyor: set EPICS_HOST_ARCH --- appveyor/do.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/appveyor/do.py b/appveyor/do.py index 6ef4793..744e869 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -295,7 +295,7 @@ def prepare(*args): source_set(os.environ['SET']) [complete_setup(mod) for mod in modlist()] - + logger.debug('Loaded setup') kvs = list(setup.items()) kvs.sort() @@ -327,7 +327,13 @@ def prepare(*args): else: optitype = 'optimized' - print('EPICS Base set up for {0} build with {1} linking'.format(optitype, linktype)) + if os.environ['PLATFORM'] == 'x86': + os.environ['EPICS_HOST_ARCH'] = 'win32-x86' + elif os.environ['PLATFORM'] == 'x64': + os.environ['EPICS_HOST_ARCH'] = 'windows-x64' + + print('EPICS Base set up on {0} for {1} build with {2} linking' + .format(os.environ['EPICS_HOST_ARCH'], optitype, linktype)) if not os.path.isdir(toolsdir): os.makedirs(toolsdir) @@ -361,7 +367,6 @@ def prepare(*args): def build(*args): print('{0}Building the module{1}'.format(ANSI_YELLOW, ANSI_RESET)) - def test(*args): print('Running the tests') From a88300bdd762e9f617f4a746aa024738c6f544d2 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Mon, 23 Mar 2020 14:55:36 +0100 Subject: [PATCH 24/68] appveyor: put strawberry perl in the PATH --- appveyor/do.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/appveyor/do.py b/appveyor/do.py index 744e869..d5f1145 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -37,9 +37,6 @@ cachedir = os.path.join('.', '.cache') toolsdir = os.path.join('.', '.tools') -# ensure our 'make' found first -os.environ['PATH'] = os.pathsep.join([toolsdir, os.environ['PATH']]) - def modlist(): add_modules = os.environ.get('ADD_MODULES', '').upper().split() @@ -346,6 +343,9 @@ def prepare(*args): cwd=toolsdir) sp.check_call([zip7, 'e', 'make-4.2.1.zip'], cwd=toolsdir) + # put our 'make' in the PATH + os.environ['PATH'] = os.pathsep.join([toolsdir, os.environ['PATH']]) + perlver = '5.30.0.1' if os.environ['CC'] == 'vs2019': print('Installing Strawberry Perl {0}'.format(perlver)) @@ -359,6 +359,11 @@ def prepare(*args): sp.check_call('relocation.pl.bat', shell=True, cwd=os.path.join(toolsdir, 'strawberry')) + # put our strawberry 'perl' in the PATH + os.environ['PATH'] = os.pathsep.join([os.path.join(toolsdir, 'strawberry', 'perl', 'site', 'bin'), + os.path.join(toolsdir, 'strawberry', 'perl', 'bin'), + os.environ['PATH']]) + for mod in modlist(): place = places[setup[mod+"_VARNAME"]] print('Building '+place) From c056b5ad0fa3214d2b97eb3360ebb01a9163d83c Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Mon, 23 Mar 2020 15:01:21 +0100 Subject: [PATCH 25/68] appveyor: only grep 'vcvarsall.bat' in VS install list --- appveyor/do.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor/do.py b/appveyor/do.py index d5f1145..55c9e2c 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -59,7 +59,7 @@ def host_info(): for base in (r'C:\Program Files (x86)', r'C:\Program Files'): for root, dirs, files in os.walk(base): for fname in files: - if fnmatch(fname, 'vcvars*.bat'): + if fnmatch(fname, 'vcvarsall.bat'): print('Found', os.path.join(root, fname)) # Used from unittests From 430a699d7f9c173e034f3fd4a7fbaa90d5d9ed0e Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Mon, 23 Mar 2020 16:06:31 +0100 Subject: [PATCH 26/68] appveyor: add mingw definitions --- appveyor/do.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/appveyor/do.py b/appveyor/do.py index 55c9e2c..e655c40 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -329,7 +329,7 @@ def prepare(*args): elif os.environ['PLATFORM'] == 'x64': os.environ['EPICS_HOST_ARCH'] = 'windows-x64' - print('EPICS Base set up on {0} for {1} build with {2} linking' + print('EPICS Base build system set up on {0} for {1} build with {2} linking' .format(os.environ['EPICS_HOST_ARCH'], optitype, linktype)) if not os.path.isdir(toolsdir): @@ -343,8 +343,7 @@ def prepare(*args): cwd=toolsdir) sp.check_call([zip7, 'e', 'make-4.2.1.zip'], cwd=toolsdir) - # put our 'make' in the PATH - os.environ['PATH'] = os.pathsep.join([toolsdir, os.environ['PATH']]) + make = os.path.join(toolsdir, 'make') perlver = '5.30.0.1' if os.environ['CC'] == 'vs2019': @@ -363,11 +362,26 @@ def prepare(*args): os.environ['PATH'] = os.pathsep.join([os.path.join(toolsdir, 'strawberry', 'perl', 'site', 'bin'), os.path.join(toolsdir, 'strawberry', 'perl', 'bin'), os.environ['PATH']]) + if os.environ['CC'] == 'mingw': + if 'INCLUDE' not in os.environ: + os.environ['INCLUDE'] = '' + if os.environ['PLATFORM'] == 'x86': + os.environ['EPICS_HOST_ARCH'] = 'win32-x86-mingw' + os.environ['INCLUDE'] = os.pathsep.join(['C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\include', + os.environ['INCLUDE']]) + os.environ['PATH'] = os.pathsep.join(['C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin', + os.environ['PATH']]) + elif os.environ['PLATFORM'] == 'x64': + os.environ['EPICS_HOST_ARCH'] = 'windows-x64-mingw' + os.environ['INCLUDE'] = os.pathsep.join(['C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\include', + os.environ['INCLUDE']]) + os.environ['PATH'] = os.pathsep.join(['C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin', + os.environ['PATH']]) for mod in modlist(): place = places[setup[mod+"_VARNAME"]] print('Building '+place) - sp.check_call('make', shell=True, cwd=place) + sp.check_call(make, shell=True, cwd=place) def build(*args): print('{0}Building the module{1}'.format(ANSI_YELLOW, ANSI_RESET)) From 24a70882d0ff0642c0b2c033bc6d6b03820eb0aa Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Mon, 23 Mar 2020 16:07:03 +0100 Subject: [PATCH 27/68] appveyor: use parallel make (except for Base 3.14) --- appveyor/do.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/appveyor/do.py b/appveyor/do.py index e655c40..72a85b3 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -345,6 +345,12 @@ def prepare(*args): make = os.path.join(toolsdir, 'make') + makeargs = ['-j2', '-Otarget'] + # no parallel make for Base 3.14 + with open(os.path.join(places[setup['BASE_VARNAME']], 'configure', 'CONFIG_BASE_VERSION')) as myfile: + if 'BASE_3_14=YES' in myfile.read(): + makeargs = [] + perlver = '5.30.0.1' if os.environ['CC'] == 'vs2019': print('Installing Strawberry Perl {0}'.format(perlver)) @@ -381,7 +387,7 @@ def prepare(*args): for mod in modlist(): place = places[setup[mod+"_VARNAME"]] print('Building '+place) - sp.check_call(make, shell=True, cwd=place) + sp.check_call([make] + makeargs, shell=True, cwd=place) def build(*args): print('{0}Building the module{1}'.format(ANSI_YELLOW, ANSI_RESET)) From e11632798a2b1b1e463548c5064813d3ff99ee44 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Mon, 23 Mar 2020 16:13:04 +0100 Subject: [PATCH 28/68] appveyor: print make version --- appveyor/do.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/appveyor/do.py b/appveyor/do.py index 72a85b3..0b5a263 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -19,6 +19,8 @@ ANSI_GREEN = "\033[32;1m" ANSI_YELLOW = "\033[33;1m" ANSI_BLUE = "\033[34;1m" +ANSI_MAGENTA = "\033[35;1m" +ANSI_CYAN = "\033[36;1m" ANSI_RESET = "\033[0m" ANSI_CLEAR = "\033[0K" @@ -383,6 +385,9 @@ def prepare(*args): os.environ['INCLUDE']]) os.environ['PATH'] = os.pathsep.join(['C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin', os.environ['PATH']]) + print('{0}$ {1} --version{2}'.format(ANSI_CYAN, make, ANSI_RESET)) + sys.stdout.flush() + sp.check_call([make, '--version']) for mod in modlist(): place = places[setup[mod+"_VARNAME"]] From 0ae628673cc48875341eb69f143a8fc9793a3dc0 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Mon, 23 Mar 2020 17:01:40 +0100 Subject: [PATCH 29/68] appveyor: reduce number of SET=test00 runs --- .appveyor.yml | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index c9ba9a6..f202dc2 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -70,19 +70,18 @@ platform: # Matrix configuration: allow specific failing jobs matrix: exclude: + # Run test00 only once: x64 dynamic + - platform: x86 + SET: test00 + - configuration: static + SET: test00 + - configuration: dynamic-debug + SET: test00 + - configuration: static-debug + SET: test00 # VS2010 Express installs don't have the 64 bit compiler - platform: x64 CC: vs2010 - # Exclude to reduce total job runtime - # skip 64-bit for older and 32-bit for newer - - platform: x64 - CC: vs2012 - - platform: x86 - CC: mingw - - platform: x86 - CC: vs2019 - - platform: x86 - CC: vs2017 #---------------------------------# From 11cb469fb927d91150cec1ab97b0018050ac68c7 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Mon, 23 Mar 2020 17:05:30 +0100 Subject: [PATCH 30/68] appveyor: silence Perl relocation batch script --- appveyor/do.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/appveyor/do.py b/appveyor/do.py index 0b5a263..b98cd16 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -363,8 +363,9 @@ def prepare(*args): cwd=toolsdir) sp.check_call([zip7, 'x', 'perl-{0}.zip'.format(perlver), '-ostrawberry'], cwd=toolsdir) - sp.check_call('relocation.pl.bat', shell=True, - cwd=os.path.join(toolsdir, 'strawberry')) + with open(os.devnull, 'w') as devnull: + sp.check_call('relocation.pl.bat', shell=True, stdout=devnull, + cwd=os.path.join(toolsdir, 'strawberry')) # put our strawberry 'perl' in the PATH os.environ['PATH'] = os.pathsep.join([os.path.join(toolsdir, 'strawberry', 'perl', 'site', 'bin'), From ff14d5ceb4abdda09d914d75ec1faa97e5b1d150 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Mon, 23 Mar 2020 17:54:38 +0100 Subject: [PATCH 31/68] appveyor: adding make calls for 'build' and 'test' actions --- appveyor/do.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/appveyor/do.py b/appveyor/do.py index b98cd16..e4d283e 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -48,6 +48,8 @@ def modlist(): return ret zip7 = 'C:\\Program Files\\7-Zip\\7z' +make = '' +makeargs = [] def host_info(): print(sys.version) @@ -393,13 +395,15 @@ def prepare(*args): for mod in modlist(): place = places[setup[mod+"_VARNAME"]] print('Building '+place) - sp.check_call([make] + makeargs, shell=True, cwd=place) + sp.check_call([make] + makeargs, cwd=place) def build(*args): print('{0}Building the module{1}'.format(ANSI_YELLOW, ANSI_RESET)) + sp.check_call([make] + makeargs) def test(*args): - print('Running the tests') + print('{0}Running the tests{1}'.format(ANSI_YELLOW, ANSI_RESET)) + sp.check_call([make] + makeargs + ['tapfiles']) def doExec(*args): 'exec user command with vcvars' From 11f5c94236ed2319d5e57bf52fc2255e43afc573 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Tue, 24 Mar 2020 11:11:10 +0100 Subject: [PATCH 32/68] appveyor: use r'' string constants for all Windows paths --- appveyor/do.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/appveyor/do.py b/appveyor/do.py index e4d283e..5113ea1 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -47,7 +47,7 @@ def modlist(): logger.debug('Effective module list: %s', ret) return ret -zip7 = 'C:\\Program Files\\7-Zip\\7z' +zip7 = r'C:\Program Files\7-Zip\7z' make = '' makeargs = [] @@ -378,15 +378,15 @@ def prepare(*args): os.environ['INCLUDE'] = '' if os.environ['PLATFORM'] == 'x86': os.environ['EPICS_HOST_ARCH'] = 'win32-x86-mingw' - os.environ['INCLUDE'] = os.pathsep.join(['C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\include', + os.environ['INCLUDE'] = os.pathsep.join([r'C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\include', os.environ['INCLUDE']]) - os.environ['PATH'] = os.pathsep.join(['C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin', + os.environ['PATH'] = os.pathsep.join([r'C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin', os.environ['PATH']]) elif os.environ['PLATFORM'] == 'x64': os.environ['EPICS_HOST_ARCH'] = 'windows-x64-mingw' - os.environ['INCLUDE'] = os.pathsep.join(['C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\include', + os.environ['INCLUDE'] = os.pathsep.join([r'C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\include', os.environ['INCLUDE']]) - os.environ['PATH'] = os.pathsep.join(['C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin', + os.environ['PATH'] = os.pathsep.join([r'C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin', os.environ['PATH']]) print('{0}$ {1} --version{2}'.format(ANSI_CYAN, make, ANSI_RESET)) sys.stdout.flush() From 421fe54fe68a1cd94f1b49454375930cf4c93335 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Fri, 27 Mar 2020 10:24:06 +0100 Subject: [PATCH 33/68] appveyor: move environment setting into setup_for_build() --- appveyor/do.py | 78 +++++++++++++++++++++++++++----------------------- 1 file changed, 42 insertions(+), 36 deletions(-) diff --git a/appveyor/do.py b/appveyor/do.py index 5113ea1..89f468d 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -287,6 +287,41 @@ def add_dependency(dep): update_release_local(setup[dep+"_VARNAME"], place) +def setup_for_build(): + global make, makeargs + make = os.path.join(toolsdir, 'make') + makeargs = ['-j2', '-Otarget'] + # no parallel make for Base 3.14 + with open(os.path.join(places[setup['BASE_VARNAME']], 'configure', 'CONFIG_BASE_VERSION')) as myfile: + if 'BASE_3_14=YES' in myfile.read(): + makeargs = [] + + if os.environ['PLATFORM'] == 'x86': + os.environ['EPICS_HOST_ARCH'] = 'win32-x86' + elif os.environ['PLATFORM'] == 'x64': + os.environ['EPICS_HOST_ARCH'] = 'windows-x64' + + if os.environ['CC'] == 'vs2019': + # put our strawberry 'perl' in the PATH + os.environ['PATH'] = os.pathsep.join([os.path.join(toolsdir, 'strawberry', 'perl', 'site', 'bin'), + os.path.join(toolsdir, 'strawberry', 'perl', 'bin'), + os.environ['PATH']]) + if os.environ['CC'] == 'mingw': + if 'INCLUDE' not in os.environ: + os.environ['INCLUDE'] = '' + if os.environ['PLATFORM'] == 'x86': + os.environ['EPICS_HOST_ARCH'] = 'win32-x86-mingw' + os.environ['INCLUDE'] = os.pathsep.join([r'C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\include', + os.environ['INCLUDE']]) + os.environ['PATH'] = os.pathsep.join([r'C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin', + os.environ['PATH']]) + elif os.environ['PLATFORM'] == 'x64': + os.environ['EPICS_HOST_ARCH'] = 'windows-x64-mingw' + os.environ['INCLUDE'] = os.pathsep.join([r'C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\include', + os.environ['INCLUDE']]) + os.environ['PATH'] = os.pathsep.join([r'C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin', + os.environ['PATH']]) + def prepare(*args): host_info() @@ -328,38 +363,23 @@ def prepare(*args): else: optitype = 'optimized' - if os.environ['PLATFORM'] == 'x86': - os.environ['EPICS_HOST_ARCH'] = 'win32-x86' - elif os.environ['PLATFORM'] == 'x64': - os.environ['EPICS_HOST_ARCH'] = 'windows-x64' - - print('EPICS Base build system set up on {0} for {1} build with {2} linking' - .format(os.environ['EPICS_HOST_ARCH'], optitype, linktype)) + print('EPICS Base build system set up for {0} build with {1} linking' + .format(optitype, linktype)) if not os.path.isdir(toolsdir): os.makedirs(toolsdir) print('Installing Make 4.2.1 from ANL web site') sys.stdout.flush() - sp.check_call(['curl', '-fsS', '--retry', '3', '-o', 'make-4.2.1.zip', 'https://epics.anl.gov/download/tools/make-4.2.1-win64.zip'], cwd=toolsdir) sp.check_call([zip7, 'e', 'make-4.2.1.zip'], cwd=toolsdir) - make = os.path.join(toolsdir, 'make') - - makeargs = ['-j2', '-Otarget'] - # no parallel make for Base 3.14 - with open(os.path.join(places[setup['BASE_VARNAME']], 'configure', 'CONFIG_BASE_VERSION')) as myfile: - if 'BASE_3_14=YES' in myfile.read(): - makeargs = [] - perlver = '5.30.0.1' if os.environ['CC'] == 'vs2019': print('Installing Strawberry Perl {0}'.format(perlver)) sys.stdout.flush() - sp.check_call(['curl', '-fsS', '--retry', '3', '-o', 'perl-{0}.zip'.format(perlver), 'http://strawberryperl.com/download/{0}/strawberry-perl-{0}-64bit.zip'.format(perlver)], cwd=toolsdir) @@ -369,25 +389,8 @@ def prepare(*args): sp.check_call('relocation.pl.bat', shell=True, stdout=devnull, cwd=os.path.join(toolsdir, 'strawberry')) - # put our strawberry 'perl' in the PATH - os.environ['PATH'] = os.pathsep.join([os.path.join(toolsdir, 'strawberry', 'perl', 'site', 'bin'), - os.path.join(toolsdir, 'strawberry', 'perl', 'bin'), - os.environ['PATH']]) - if os.environ['CC'] == 'mingw': - if 'INCLUDE' not in os.environ: - os.environ['INCLUDE'] = '' - if os.environ['PLATFORM'] == 'x86': - os.environ['EPICS_HOST_ARCH'] = 'win32-x86-mingw' - os.environ['INCLUDE'] = os.pathsep.join([r'C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\include', - os.environ['INCLUDE']]) - os.environ['PATH'] = os.pathsep.join([r'C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin', - os.environ['PATH']]) - elif os.environ['PLATFORM'] == 'x64': - os.environ['EPICS_HOST_ARCH'] = 'windows-x64-mingw' - os.environ['INCLUDE'] = os.pathsep.join([r'C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\include', - os.environ['INCLUDE']]) - os.environ['PATH'] = os.pathsep.join([r'C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin', - os.environ['PATH']]) + setup_for_build() + print('{0}$ {1} --version{2}'.format(ANSI_CYAN, make, ANSI_RESET)) sys.stdout.flush() sp.check_call([make, '--version']) @@ -398,15 +401,18 @@ def prepare(*args): sp.check_call([make] + makeargs, cwd=place) def build(*args): + setup_for_build() print('{0}Building the module{1}'.format(ANSI_YELLOW, ANSI_RESET)) sp.check_call([make] + makeargs) def test(*args): + setup_for_build() print('{0}Running the tests{1}'.format(ANSI_YELLOW, ANSI_RESET)) sp.check_call([make] + makeargs + ['tapfiles']) def doExec(*args): 'exec user command with vcvars' + setup_for_build() print('Execute command {}'.format(args)) sp.check_call(' '.join(args), shell=True) From 9c58196b6d4b0225bb886518ea90898f4dec1bcc Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Fri, 27 Mar 2020 11:37:22 +0100 Subject: [PATCH 34/68] appveyor: use Base 3.15 for tests to speed up build --- .appveyor.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.appveyor.yml b/.appveyor.yml index f202dc2..28dc6ec 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -46,6 +46,7 @@ environment: # common variables SETUP_PATH: .:.ci SET: test01 + BASE: 3.15 matrix: - CC: vs2019 From b3efae245121675ec467e0c2ce21a2e5858812ba Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Fri, 27 Mar 2020 10:24:19 +0100 Subject: [PATCH 35/68] appveyor: improve logging / print formatting --- appveyor/do.py | 57 ++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 44 insertions(+), 13 deletions(-) diff --git a/appveyor/do.py b/appveyor/do.py index 89f468d..f3c276c 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -52,19 +52,21 @@ def modlist(): makeargs = [] def host_info(): + print('{0}Python setup{1}'.format(ANSI_CYAN, ANSI_RESET)) print(sys.version) print('PYTHONPATH') for dname in sys.path: print(' ', dname) - print('platform = ', distutils.util.get_platform()) + print('platform =', distutils.util.get_platform()) - print('Listing available VS versions') + print('{0}Available Visual Studio versions{1}'.format(ANSI_CYAN, ANSI_RESET)) from fnmatch import fnmatch for base in (r'C:\Program Files (x86)', r'C:\Program Files'): for root, dirs, files in os.walk(base): for fname in files: if fnmatch(fname, 'vcvarsall.bat'): print('Found', os.path.join(root, fname)) + sys.stdout.flush() # Used from unittests def clear_lists(): @@ -98,6 +100,7 @@ def source_set(name): if os.path.isfile(set_file): seen_setups.append(set_file) print("Loading setup file {0}".format(set_file)) + sys.stdout.flush() with open(set_file) as fp: for line in fp: logger.debug('Next line: %s', line.strip()) @@ -172,7 +175,11 @@ def set_setup_from_env(dep): logger.debug('ENV assignment: %s = %s', dep+postf, setup[dep+postf]) def call_git(args, **kws): - logger.debug("EXEC '%s' in %s", ' '.join(['git'] + args), os.getcwd()) + if 'cwd' in kws: + place = kws['cwd'] + else: + place = os.getcwd() + logger.debug("EXEC '%s' in %s", ' '.join(['git'] + args), place) sys.stdout.flush() exitcode = sp.call(['git'] + args, **kws) logger.debug('EXEC DONE') @@ -247,6 +254,7 @@ def add_dependency(dep): shutil.rmtree(place, onerror=remove_readonly) else: print('Found {0} of dependency {1} up-to-date in {2}'.format(tag, dep, place)) + sys.stdout.flush() if not os.path.isdir(place): if not os.path.isdir(cachedir): @@ -258,9 +266,10 @@ def add_dependency(dep): }.get(setup[dep+'_DEPTH'], ['--depth', setup[dep+'_DEPTH']]) print('Cloning {0} of dependency {1} into {2}' .format(tag, dep, place)) + sys.stdout.flush() call_git(['clone', '--quiet'] + deptharg + [recursearg, '--branch', tag, setup[dep+'_REPOURL'], dirname], cwd=cachedir) - sp.check_call(['git', 'log', '-n1']) + sp.check_call(['git', 'log', '-n1'], cwd=place) modules_to_compile.append(place) # force including RELEASE.local for non-base modules by overwriting their configure/RELEASE @@ -275,7 +284,7 @@ def add_dependency(dep): hook = os.path.join(place, setup[dep+'_HOOK']) if os.path.exists(hook): print('Running hook {0} in {1}'.format(setup[dep+'_HOOK'], place)) - + sys.stdout.flush() sp.check_call(hook, shell=True, cwd=place) # write checked out commit hash to marker file @@ -341,6 +350,7 @@ def prepare(*args): call_git(['config', '--global', 'advice.detachedHead', 'false']) print('{0}Checking/cloning dependencies{1}'.format(ANSI_YELLOW, ANSI_RESET)) + sys.stdout.flush() [add_dependency(mod) for mod in modlist()] @@ -348,7 +358,7 @@ def prepare(*args): release_local = os.path.join(cachedir, 'RELEASE.local') shutil.copy(release_local, 'configure') - print('{0}Setting up EPICS build system{1}'.format(ANSI_YELLOW, ANSI_RESET)) + print('{0}Configuring EPICS build system{1}'.format(ANSI_YELLOW, ANSI_RESET)) with open(os.path.join(places['EPICS_BASE'], 'configure', 'CONFIG_SITE'), 'a') as config_site: if re.search('static', os.environ['CONFIGURATION']): @@ -391,30 +401,47 @@ def prepare(*args): setup_for_build() + print('{0}EPICS_HOST_ARCH = {1}{2}'.format(ANSI_CYAN, os.environ['EPICS_HOST_ARCH'], ANSI_RESET)) + print('{0}make arguments = {1}{2}'.format(ANSI_CYAN, ' '.join(makeargs), ANSI_RESET)) print('{0}$ {1} --version{2}'.format(ANSI_CYAN, make, ANSI_RESET)) sys.stdout.flush() sp.check_call([make, '--version']) + print('{0}$ perl --version{1}'.format(ANSI_CYAN, ANSI_RESET)) + sys.stdout.flush() + sp.check_call(['perl', '--version']) + + if os.environ['CC'] == 'mingw': + print('{0}$ gcc --version{1}'.format(ANSI_CYAN, ANSI_RESET)) + sys.stdout.flush() + sp.check_call(['gcc', '--version']) + else: + print('{0}$ cl{1}'.format(ANSI_CYAN, ANSI_RESET)) + sys.stdout.flush() + sp.check_call(['cl']) for mod in modlist(): place = places[setup[mod+"_VARNAME"]] - print('Building '+place) + print('{0}Building dependency {1} in {2}{3}'.format(ANSI_YELLOW, mod, place, ANSI_RESET)) + sys.stdout.flush() sp.check_call([make] + makeargs, cwd=place) def build(*args): setup_for_build() - print('{0}Building the module{1}'.format(ANSI_YELLOW, ANSI_RESET)) + print('{0}Building the main module{1}'.format(ANSI_YELLOW, ANSI_RESET)) + sys.stdout.flush() sp.check_call([make] + makeargs) def test(*args): setup_for_build() - print('{0}Running the tests{1}'.format(ANSI_YELLOW, ANSI_RESET)) + print('{0}Running the main module tests{1}'.format(ANSI_YELLOW, ANSI_RESET)) + sys.stdout.flush() sp.check_call([make] + makeargs + ['tapfiles']) def doExec(*args): 'exec user command with vcvars' setup_for_build() print('Execute command {}'.format(args)) - + sys.stdout.flush() sp.check_call(' '.join(args), shell=True) def with_vcvars(cmd): @@ -452,12 +479,16 @@ def with_vcvars(cmd): "{python}" "{self}" {cmd} '''.format(**info) - print('vcvars-trampoline.bat') - print(script) + logger.debug('----- Creating vcvars-trampoline.bat -----') + for line in script.split('\n'): + logger.debug(line) + logger.debug('----- snip -----') with open('vcvars-trampoline.bat', 'w') as F: F.write(script) + print('{0}Calling vcvars-trampoline.bat to set environment for {1} on {2}{3}' + .format(ANSI_YELLOW, CC, os.environ['PLATFORM'], ANSI_RESET)) sys.stdout.flush() sp.check_call('vcvars-trampoline.bat', shell=True) @@ -477,5 +508,5 @@ def with_vcvars(cmd): else: name = args.pop(0) - print('IN', name, 'with', args) + logger.debug('DO running action %s with %s', name, args) actions[name](*args) From eb5dd2a86c52d04e340e27288f2ec546bdad41fc Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Wed, 25 Mar 2020 10:07:27 +0100 Subject: [PATCH 36/68] appveyor: read Base location from RELEASE.local --- appveyor/do.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/appveyor/do.py b/appveyor/do.py index f3c276c..492cecc 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -298,10 +298,17 @@ def add_dependency(dep): def setup_for_build(): global make, makeargs + make = os.path.join(toolsdir, 'make') makeargs = ['-j2', '-Otarget'] # no parallel make for Base 3.14 - with open(os.path.join(places[setup['BASE_VARNAME']], 'configure', 'CONFIG_BASE_VERSION')) as myfile: + with open(os.path.join(cachedir, 'RELEASE.local'), 'r') as f: + lines = f.readlines() + for line in lines: + if re.search(r'^EPICS_BASE=', line): + base_place = line.split('=')[1].strip() + break + with open(os.path.join(base_place, 'configure', 'CONFIG_BASE_VERSION')) as myfile: if 'BASE_3_14=YES' in myfile.read(): makeargs = [] From 0ac6c96e2a08a3cf9807b3b2491ca111d239e591 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Wed, 25 Mar 2020 14:15:54 +0100 Subject: [PATCH 37/68] appveyor: add call_make() wrapper, forward returncode --- appveyor/do.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/appveyor/do.py b/appveyor/do.py index 492cecc..acf2cba 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -185,6 +185,18 @@ def call_git(args, **kws): logger.debug('EXEC DONE') return exitcode +def call_make(args=[], **kws): + if 'cwd' in kws: + place = kws['cwd'] + else: + place = os.getcwd() + logger.debug("EXEC '%s' in %s", ' '.join([make] + makeargs + args), place) + sys.stdout.flush() + exitcode = sp.call([make] + makeargs + args, **kws) + logger.debug('EXEC DONE') + if exitcode != 0: + sys.exit(exitcode) + def get_git_hash(place): logger.debug("EXEC 'git log -n1 --pretty=format:%%H' in %s", place) sys.stdout.flush() @@ -429,20 +441,17 @@ def prepare(*args): for mod in modlist(): place = places[setup[mod+"_VARNAME"]] print('{0}Building dependency {1} in {2}{3}'.format(ANSI_YELLOW, mod, place, ANSI_RESET)) - sys.stdout.flush() - sp.check_call([make] + makeargs, cwd=place) + call_make(cwd=place) def build(*args): setup_for_build() print('{0}Building the main module{1}'.format(ANSI_YELLOW, ANSI_RESET)) - sys.stdout.flush() - sp.check_call([make] + makeargs) + call_make() def test(*args): setup_for_build() print('{0}Running the main module tests{1}'.format(ANSI_YELLOW, ANSI_RESET)) - sys.stdout.flush() - sp.check_call([make] + makeargs + ['tapfiles']) + call_make(['tapfiles']) def doExec(*args): 'exec user command with vcvars' @@ -497,7 +506,9 @@ def with_vcvars(cmd): print('{0}Calling vcvars-trampoline.bat to set environment for {1} on {2}{3}' .format(ANSI_YELLOW, CC, os.environ['PLATFORM'], ANSI_RESET)) sys.stdout.flush() - sp.check_call('vcvars-trampoline.bat', shell=True) + returncode = sp.call('vcvars-trampoline.bat', shell=True) + if returncode != 0: + sys.exit(returncode) actions = { 'prepare': prepare, From 7881b3527c25e018446f3bfec8f5f648900b1c2b Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Wed, 25 Mar 2020 16:40:17 +0100 Subject: [PATCH 38/68] appveyor: fix for older vcvars.bat manipulating PLATFORM --- appveyor/do.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/appveyor/do.py b/appveyor/do.py index acf2cba..465df0e 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -324,9 +324,9 @@ def setup_for_build(): if 'BASE_3_14=YES' in myfile.read(): makeargs = [] - if os.environ['PLATFORM'] == 'x86': + if os.environ['PLATFORM'].lower() == 'x86': os.environ['EPICS_HOST_ARCH'] = 'win32-x86' - elif os.environ['PLATFORM'] == 'x64': + elif os.environ['PLATFORM'].lower() == 'x64': os.environ['EPICS_HOST_ARCH'] = 'windows-x64' if os.environ['CC'] == 'vs2019': @@ -337,13 +337,13 @@ def setup_for_build(): if os.environ['CC'] == 'mingw': if 'INCLUDE' not in os.environ: os.environ['INCLUDE'] = '' - if os.environ['PLATFORM'] == 'x86': + if os.environ['PLATFORM'].lower() == 'x86': os.environ['EPICS_HOST_ARCH'] = 'win32-x86-mingw' os.environ['INCLUDE'] = os.pathsep.join([r'C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\include', os.environ['INCLUDE']]) os.environ['PATH'] = os.pathsep.join([r'C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin', os.environ['PATH']]) - elif os.environ['PLATFORM'] == 'x64': + elif os.environ['PLATFORM'].lower() == 'x64': os.environ['EPICS_HOST_ARCH'] = 'windows-x64-mingw' os.environ['INCLUDE'] = os.pathsep.join([r'C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\include', os.environ['INCLUDE']]) @@ -476,7 +476,7 @@ def with_vcvars(cmd): info['arch'] = { 'x86':'x86', # 'amd64_x86' ?? 'x64':'amd64', - }[os.environ['PLATFORM']] # 'x86' or 'x64' + }[os.environ['PLATFORM'].lower()] # 'x86' or 'x64' info['vcvars'] = { # https://en.wikipedia.org/wiki/Microsoft_Visual_Studio#History From d9e1df23679dcea9e95abaf753c22dcf184181d7 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Wed, 25 Mar 2020 17:34:05 +0100 Subject: [PATCH 39/68] appveyor: add HOST_ARCH suffix for -debug and -static --- appveyor/do.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/appveyor/do.py b/appveyor/do.py index 465df0e..4462d42 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -324,10 +324,19 @@ def setup_for_build(): if 'BASE_3_14=YES' in myfile.read(): makeargs = [] + # there is no combined static and debug EPICS_HOST_ARCH target, + # so a combined debug and static target will appear to be just static + # but debug will have been specified in CONFIG_SITE by prepare() + hostarchsuffix='' + if re.search('debug', os.environ['CONFIGURATION']): + hostarchsuffix = '-debug' + if re.search('static', os.environ['CONFIGURATION']): + hostarchsuffix = '-static' + if os.environ['PLATFORM'].lower() == 'x86': - os.environ['EPICS_HOST_ARCH'] = 'win32-x86' + os.environ['EPICS_HOST_ARCH'] = 'win32-x86' + hostarchsuffix elif os.environ['PLATFORM'].lower() == 'x64': - os.environ['EPICS_HOST_ARCH'] = 'windows-x64' + os.environ['EPICS_HOST_ARCH'] = 'windows-x64' + hostarchsuffix if os.environ['CC'] == 'vs2019': # put our strawberry 'perl' in the PATH From e50271765fa5ccb01f4386931aa32b3893e975d3 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Fri, 27 Mar 2020 17:49:26 +0100 Subject: [PATCH 40/68] appveyor: add build cache for external tools --- .appveyor.yml | 6 ++++++ appveyor/do.py | 36 ++++++++++++++++++++---------------- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 28dc6ec..95c718a 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -8,6 +8,12 @@ # Ralph Lange # Copyright (c) 2020 ITER Organization +#---------------------------------# +# build cache # +#---------------------------------# + +cache: + - C:\Users\appveyor\.tools -> appveyor\do.py #---------------------------------# # repository cloning # diff --git a/appveyor/do.py b/appveyor/do.py index 4462d42..7071c85 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -407,25 +407,29 @@ def prepare(*args): if not os.path.isdir(toolsdir): os.makedirs(toolsdir) - print('Installing Make 4.2.1 from ANL web site') - sys.stdout.flush() - sp.check_call(['curl', '-fsS', '--retry', '3', '-o', 'make-4.2.1.zip', - 'https://epics.anl.gov/download/tools/make-4.2.1-win64.zip'], - cwd=toolsdir) - sp.check_call([zip7, 'e', 'make-4.2.1.zip'], cwd=toolsdir) - - perlver = '5.30.0.1' - if os.environ['CC'] == 'vs2019': - print('Installing Strawberry Perl {0}'.format(perlver)) + makever = '4.2.1' + if not os.path.exists(os.path.join(toolsdir, 'make.exe')): + print('Installing Make 4.2.1 from ANL web site') sys.stdout.flush() - sp.check_call(['curl', '-fsS', '--retry', '3', '-o', 'perl-{0}.zip'.format(perlver), - 'http://strawberryperl.com/download/{0}/strawberry-perl-{0}-64bit.zip'.format(perlver)], + sp.check_call(['curl', '-fsS', '--retry', '3', '-o', 'make-{0}.zip'.format(makever), + 'https://epics.anl.gov/download/tools/make-{0}-win64.zip'.format(makever)], cwd=toolsdir) - sp.check_call([zip7, 'x', 'perl-{0}.zip'.format(perlver), '-ostrawberry'], cwd=toolsdir) + sp.check_call([zip7, 'e', 'make-{0}.zip'.format(makever)], cwd=toolsdir) + os.remove(os.path.join(toolsdir, 'make-{0}.zip'.format(makever))) - with open(os.devnull, 'w') as devnull: - sp.check_call('relocation.pl.bat', shell=True, stdout=devnull, - cwd=os.path.join(toolsdir, 'strawberry')) + perlver = '5.30.0.1' + if os.environ['CC'] == 'vs2019': + if not os.path.isdir(os.path.join(toolsdir, 'strawberry')): + print('Installing Strawberry Perl {0}'.format(perlver)) + sys.stdout.flush() + sp.check_call(['curl', '-fsS', '--retry', '3', '-o', 'perl-{0}.zip'.format(perlver), + 'http://strawberryperl.com/download/{0}/strawberry-perl-{0}-64bit.zip'.format(perlver)], + cwd=toolsdir) + sp.check_call([zip7, 'x', 'perl-{0}.zip'.format(perlver), '-ostrawberry'], cwd=toolsdir) + os.remove(os.path.join(toolsdir, 'perl-{0}.zip'.format(perlver))) + with open(os.devnull, 'w') as devnull: + sp.check_call('relocation.pl.bat', shell=True, stdout=devnull, + cwd=os.path.join(toolsdir, 'strawberry')) setup_for_build() From eceeab66cfa712c38052aa07f53935cca7c343b3 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Thu, 26 Mar 2020 18:12:52 +0100 Subject: [PATCH 41/68] appveyor: test full matrix (no 64bit builds on <= vs2012) --- .appveyor.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 95c718a..4c6642a 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -68,10 +68,11 @@ environment: - CC: vs2013 - CC: vs2012 - CC: vs2010 + - CC: vs2008 # Platform: architecture platform: -# - x86 + - x86 - x64 # Matrix configuration: allow specific failing jobs @@ -86,9 +87,13 @@ matrix: SET: test00 - configuration: static-debug SET: test00 - # VS2010 Express installs don't have the 64 bit compiler + # VS2012 and older installs don't have the 64 bit compiler + - platform: x64 + CC: vs2012 - platform: x64 CC: vs2010 + - platform: x64 + CC: vs2008 #---------------------------------# From b7d505c2e2056b739da043ae685f7563ff035057 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Thu, 26 Mar 2020 16:14:28 +0100 Subject: [PATCH 42/68] appveyor: use '/' in RELEASE.local paths --- appveyor/do.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor/do.py b/appveyor/do.py index 7071c85..e848200 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -131,7 +131,7 @@ def source_set(name): # Set places[var] = location def update_release_local(var, location): release_local = os.path.join(cachedir, 'RELEASE.local') - updated_line = '{0}={1}'.format(var, location) + updated_line = '{0}={1}'.format(var, location.replace('\\', '/')) places[var] = location if not os.path.exists(release_local): From a00629346128382b471ab8f30fb83c2d47fcfb23 Mon Sep 17 00:00:00 2001 From: Michael Davidsaver Date: Thu, 26 Mar 2020 12:46:41 -0700 Subject: [PATCH 43/68] appveyor: fixup argument parsing --- appveyor-test.py | 6 +++--- appveyor/do.py | 55 ++++++++++++++++++++++++++++++------------------ 2 files changed, 38 insertions(+), 23 deletions(-) diff --git a/appveyor-test.py b/appveyor-test.py index 61f1e62..ef4a727 100644 --- a/appveyor-test.py +++ b/appveyor-test.py @@ -256,6 +256,6 @@ def test_vcvars(self): else: unittest.main() else: - do.actions['prepare']() - do.actions['build']() - do.actions['test']() + do.main(['prepare']) + do.main(['build']) + do.main(['test']) diff --git a/appveyor/do.py b/appveyor/do.py index e848200..f468fa4 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -359,7 +359,7 @@ def setup_for_build(): os.environ['PATH'] = os.pathsep.join([r'C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin', os.environ['PATH']]) -def prepare(*args): +def prepare(args): host_info() print('{0}Loading setup files{1}'.format(ANSI_YELLOW, ANSI_RESET)) @@ -456,22 +456,22 @@ def prepare(*args): print('{0}Building dependency {1} in {2}{3}'.format(ANSI_YELLOW, mod, place, ANSI_RESET)) call_make(cwd=place) -def build(*args): +def build(args): setup_for_build() print('{0}Building the main module{1}'.format(ANSI_YELLOW, ANSI_RESET)) call_make() -def test(*args): +def test(args): setup_for_build() print('{0}Running the main module tests{1}'.format(ANSI_YELLOW, ANSI_RESET)) call_make(['tapfiles']) -def doExec(*args): +def doExec(args): 'exec user command with vcvars' setup_for_build() - print('Execute command {}'.format(args)) + print('Execute command {}'.format(args.cmd)) sys.stdout.flush() - sp.check_call(' '.join(args), shell=True) + sp.check_call(' '.join(args.cmd), shell=True) def with_vcvars(cmd): '''re-exec main script with a (hopefully different) command @@ -523,21 +523,36 @@ def with_vcvars(cmd): if returncode != 0: sys.exit(returncode) -actions = { - 'prepare': prepare, - 'build': build, - 'test': test, - 'exec': doExec, - '_vcvars':lambda:None, -} +def getargs(): + from argparse import ArgumentParser, REMAINDER + P = ArgumentParser() + P.add_argument('--no-vcvars', dest='vcvars', default=True, action='store_false', + help='Assume vcvarsall.bat has already been run') + SP = P.add_subparsers() -if __name__=='__main__': - args = sys.argv[1:] - if args[0]!='_vcvars' and os.environ['CC']!='mingw': + CMD = SP.add_parser('prepare') + CMD.set_defaults(func=prepare) + + CMD = SP.add_parser('build') + CMD.set_defaults(func=build) + + CMD = SP.add_parser('test') + CMD.set_defaults(func=test) + + CMD = SP.add_parser('exec') + CMD.add_argument('cmd', nargs=REMAINDER) + CMD.set_defaults(func=doExec) + + return P + +def main(raw): + args = getargs().parse_args(raw) + if args.vcvars and os.environ['CC'].startswith('vs'): # re-exec with MSVC in PATH - with_vcvars(' '.join(['_vcvars']+args)) + with_vcvars(' '.join(['--no-vcvars']+raw)) else: - name = args.pop(0) - logger.debug('DO running action %s with %s', name, args) - actions[name](*args) + args.func(args) + +if __name__=='__main__': + main(sys.argv[1:]) From fc141f874a7274888fb52b31f763145f0c627421 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Fri, 27 Mar 2020 16:32:54 +0100 Subject: [PATCH 44/68] appveyor: run unit tests via matrix 'only' setting --- .appveyor.yml | 15 ++++++++++++++- appveyor-test.py | 15 +++++---------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 4c6642a..047555d 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -95,14 +95,27 @@ matrix: - platform: x64 CC: vs2008 +# Run test script for unit tests (SET = test00) +for: +- + matrix: + only: + - SET: test00 + build_script: + - cmd: python appveyor-test.py + test_script: + - cmd: echo Tests have been run in the build phase #---------------------------------# # building & testing # #---------------------------------# build_script: - - cmd: python appveyor-test.py + - cmd: python appveyor/do.py prepare + - cmd: python appveyor/do.py build +test_script: + - cmd: python appveyor/do.py test #---------------------------------# # debugging # diff --git a/appveyor-test.py b/appveyor-test.py index ef4a727..8f782be 100644 --- a/appveyor-test.py +++ b/appveyor-test.py @@ -248,14 +248,9 @@ def test_vcvars(self): do.with_vcvars('env') if __name__ == "__main__": - if 'SET' in os.environ and os.environ['SET'] == "test00": - do.host_info() - if sys.argv[1:]==['env']: - # testing with_vcvars - [print(K,'=',V) for K, V in os.environ.items()] - else: - unittest.main() + do.host_info() + if sys.argv[1:]==['env']: + # testing with_vcvars + [print(K,'=',V) for K, V in os.environ.items()] else: - do.main(['prepare']) - do.main(['build']) - do.main(['test']) + unittest.main() From 5d4fdec627379389501d268fd3c7b16c339e544e Mon Sep 17 00:00:00 2001 From: Michael Davidsaver Date: Fri, 27 Mar 2020 14:25:33 -0700 Subject: [PATCH 45/68] appveyor: add newlines when writing to CONFIG_SITE --- appveyor/do.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/appveyor/do.py b/appveyor/do.py index f468fa4..4494b04 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -390,13 +390,13 @@ def prepare(args): with open(os.path.join(places['EPICS_BASE'], 'configure', 'CONFIG_SITE'), 'a') as config_site: if re.search('static', os.environ['CONFIGURATION']): - config_site.write('SHARED_LIBRARIES=NO') - config_site.write('STATIC_BUILD=YES') + config_site.write('SHARED_LIBRARIES=NO\n') + config_site.write('STATIC_BUILD=YES\n') linktype = 'static' else: linktype = 'dynamic (DLL)' if re.search('debug', os.environ['CONFIGURATION']): - config_site.write('HOST_OPT=NO') + config_site.write('HOST_OPT=NO\n') optitype = 'debug' else: optitype = 'optimized' From e14b97b18ed7f199a2cb2338407a0c3059d53d36 Mon Sep 17 00:00:00 2001 From: Michael Davidsaver Date: Sat, 28 Mar 2020 13:36:46 -0700 Subject: [PATCH 46/68] appveyor: fix _RECURSIVE option --- appveyor/do.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/appveyor/do.py b/appveyor/do.py index 4494b04..ffa7616 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -214,7 +214,7 @@ def complete_setup(dep): setup.setdefault(dep+"_REPOURL", 'https://github.com/{0}/{1}.git' .format(setup[dep+'_REPOOWNER'], setup[dep+'_REPONAME'])) setup.setdefault(dep+"_VARNAME", dep) - setup.setdefault(dep+"_RECURSIVE", 1) + setup.setdefault(dep+"_RECURSIVE", 'YES') setup.setdefault(dep+"_DEPTH", -1) # add_dependency(dep, tag) @@ -232,10 +232,13 @@ def complete_setup(dep): # - Add $dep_VARNAME line to the RELEASE.local file in the cache area (unless already there) # - Add full path to $modules_to_compile def add_dependency(dep): - if setup[dep+'_RECURSIVE'] not in [0, 'no']: - recursearg = "--recursive" + recurse = setup[dep+'_RECURSIVE'].lower() + if recurse not in ['0', 'no']: + recursearg = ["--recursive"] + elif recurse not in ['1', 'yes']: + recursearg = [] else: - recursearg = '' + raise RuntimeError("Invalid value for {}_RECURSIVE='{}' not 0/NO/1/YES".format(dep, recurse)) tag = setup[dep] @@ -279,7 +282,7 @@ def add_dependency(dep): print('Cloning {0} of dependency {1} into {2}' .format(tag, dep, place)) sys.stdout.flush() - call_git(['clone', '--quiet'] + deptharg + [recursearg, '--branch', tag, setup[dep+'_REPOURL'], dirname], cwd=cachedir) + call_git(['clone', '--quiet'] + deptharg + recursearg + ['--branch', tag, setup[dep+'_REPOURL'], dirname], cwd=cachedir) sp.check_call(['git', 'log', '-n1'], cwd=place) modules_to_compile.append(place) From e96f77d8b1b48713f89c279b6cdbeda047ea0080 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Mon, 30 Mar 2020 18:02:02 +0200 Subject: [PATCH 47/68] appveyor: fix _DEPTH option --- appveyor/do.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/appveyor/do.py b/appveyor/do.py index ffa7616..2ffa882 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -239,6 +239,10 @@ def add_dependency(dep): recursearg = [] else: raise RuntimeError("Invalid value for {}_RECURSIVE='{}' not 0/NO/1/YES".format(dep, recurse)) + deptharg = { + '-1':['--depth', '5'], + '0':[], + }.get(str(setup[dep+'_DEPTH']), ['--depth', str(setup[dep+'_DEPTH'])]) tag = setup[dep] @@ -275,10 +279,6 @@ def add_dependency(dep): if not os.path.isdir(cachedir): os.makedirs(cachedir) # clone dependency - deptharg = { - -1:['--depth', '5'], - 0:[], - }.get(setup[dep+'_DEPTH'], ['--depth', setup[dep+'_DEPTH']]) print('Cloning {0} of dependency {1} into {2}' .format(tag, dep, place)) sys.stdout.flush() From 2f8f4e7fb18e54acd1e2469f75f7ceb324d61555 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Mon, 30 Mar 2020 17:39:02 +0200 Subject: [PATCH 48/68] appveyor: add tests for _RECURSIVE and _DEPTH --- appveyor-test.py | 49 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/appveyor-test.py b/appveyor-test.py index 8f782be..76fbdec 100644 --- a/appveyor-test.py +++ b/appveyor-test.py @@ -10,6 +10,7 @@ import sys, os, shutil, fileinput import distutils.util import re +import subprocess as sp import unittest builddir = os.getcwd() @@ -155,7 +156,7 @@ def test_SetBaseAndMultipleModules(self): self.assertGreater(foundat['mod2'], foundat['mod1'], 'MOD2 (line {0}) appears before MOD1 (line {1})'.format(foundat['mod2'], foundat['mod1'])) -class TestAddDependency(unittest.TestCase): +class TestAddDependencyUpToDateCheck(unittest.TestCase): hash_3_15_6 = "ce7943fb44beb22b453ddcc0bda5398fadf72096" location = os.path.join(do.cachedir, 'base-R3.15.6') @@ -209,6 +210,52 @@ def test_OutdatedDependency(self): "Wrong commit of dependency checked out (expected='{0}' found='{1}')" .format(self.hash_3_15_6, checked_out)) +def is_shallow_repo(place): + check = sp.check_output(['git', 'rev-parse', '--is-shallow-repository'], cwd=place).strip() + if check == '--is-shallow-repository': + if os.path.exists(os.path.join(place, '.git', 'shallow')): + check = 'true' + else: + check = 'false' + return check == 'true' + +class TestAddDependencyOptions(unittest.TestCase): + + location = os.path.join(do.cachedir, 'mcoreutils-master') + testfile = os.path.join(location, '.ci', 'LICENSE') + + def setUp(self): + os.environ['SETUP_PATH'] = '.:appveyor' + if os.path.exists(self.location): + shutil.rmtree(self.location, onerror=do.remove_readonly) + do.clear_lists() + do.source_set('defaults') + do.complete_setup('MCoreUtils') + do.setup['MCoreUtils'] = 'master' + + def test_Default(self): + do.add_dependency('MCoreUtils') + self.assertTrue(os.path.exists(self.testfile), + 'Submodule (.ci) not checked out recursively (requested: default=YES') + self.assertTrue(is_shallow_repo(self.location), + 'Module not checked out shallow (requested: default=5)') + + def test_SetRecursiveNo(self): + do.setup['MCoreUtils_RECURSIVE'] = 'NO' + do.add_dependency('MCoreUtils') + self.assertFalse(os.path.exists(self.testfile), 'Submodule (.ci) checked out recursively') + + def test_SetDepthZero(self): + do.setup['MCoreUtils_DEPTH'] = '0' + do.add_dependency('MCoreUtils') + self.assertFalse(is_shallow_repo(self.location), 'Module checked out shallow (requested full)') + + def test_SetDepthThree(self): + do.setup['MCoreUtils_DEPTH'] = '3' + do.add_dependency('MCoreUtils') + self.assertTrue(is_shallow_repo(self.location), + 'Module not checked out shallow (requested: default=5)') + def repo_access(dep): do.set_setup_from_env(dep) do.setup.setdefault(dep + "_DIRNAME", dep.lower()) From dbfba732fd7250a0e7f52957ca26fb4a35fc3b92 Mon Sep 17 00:00:00 2001 From: Michael Davidsaver Date: Sun, 29 Mar 2020 20:59:08 -0700 Subject: [PATCH 49/68] appveyor: build action accepts arguments (passed on to make) --- appveyor/do.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/appveyor/do.py b/appveyor/do.py index 2ffa882..90630b8 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -462,7 +462,7 @@ def prepare(args): def build(args): setup_for_build() print('{0}Building the main module{1}'.format(ANSI_YELLOW, ANSI_RESET)) - call_make() + call_make(args.makeargs) def test(args): setup_for_build() @@ -537,6 +537,7 @@ def getargs(): CMD.set_defaults(func=prepare) CMD = SP.add_parser('build') + CMD.add_argument('makeargs', nargs=REMAINDER) CMD.set_defaults(func=build) CMD = SP.add_parser('test') From 48832354da2ff83d6e4acd39573e9fe537d0b8fe Mon Sep 17 00:00:00 2001 From: Michael Davidsaver Date: Sun, 29 Mar 2020 21:24:41 -0700 Subject: [PATCH 50/68] appveyor: export MAKE executable to environment Save user scripts from having to compute --- appveyor/do.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/appveyor/do.py b/appveyor/do.py index 90630b8..89044e3 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -314,7 +314,7 @@ def add_dependency(dep): def setup_for_build(): global make, makeargs - make = os.path.join(toolsdir, 'make') + make = os.path.join(toolsdir, 'make.exe') makeargs = ['-j2', '-Otarget'] # no parallel make for Base 3.14 with open(os.path.join(cachedir, 'RELEASE.local'), 'r') as f: @@ -472,6 +472,7 @@ def test(args): def doExec(args): 'exec user command with vcvars' setup_for_build() + os.environ['MAKE'] = make print('Execute command {}'.format(args.cmd)) sys.stdout.flush() sp.check_call(' '.join(args.cmd), shell=True) From a8321aff92615f635defbc9cbcac5bb5e1afc32a Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Wed, 1 Apr 2020 19:01:16 +0200 Subject: [PATCH 51/68] appveyor: add binary locations to PATH for tools and DLLs --- appveyor/do.py | 37 ++++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/appveyor/do.py b/appveyor/do.py index 89044e3..4a25e9c 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -313,19 +313,7 @@ def add_dependency(dep): def setup_for_build(): global make, makeargs - - make = os.path.join(toolsdir, 'make.exe') - makeargs = ['-j2', '-Otarget'] - # no parallel make for Base 3.14 - with open(os.path.join(cachedir, 'RELEASE.local'), 'r') as f: - lines = f.readlines() - for line in lines: - if re.search(r'^EPICS_BASE=', line): - base_place = line.split('=')[1].strip() - break - with open(os.path.join(base_place, 'configure', 'CONFIG_BASE_VERSION')) as myfile: - if 'BASE_3_14=YES' in myfile.read(): - makeargs = [] + dllpaths = [] # there is no combined static and debug EPICS_HOST_ARCH target, # so a combined debug and static target will appear to be just static @@ -362,6 +350,29 @@ def setup_for_build(): os.environ['PATH'] = os.pathsep.join([r'C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin', os.environ['PATH']]) + make = os.path.join(toolsdir, 'make.exe') + makeargs = ['-j2', '-Otarget'] + + with open(os.path.join(cachedir, 'RELEASE.local'), 'r') as f: + lines = f.readlines() + for line in lines: + (mod, place) = line.strip().split('=') + bindir = os.path.join(place, 'bin', os.environ['EPICS_HOST_ARCH']) + if os.path.isdir(bindir): + dllpaths.append(bindir) + if mod == 'EPICS_BASE': + base_place = place + # no parallel make for Base 3.14 + with open(os.path.join(base_place, 'configure', 'CONFIG_BASE_VERSION')) as myfile: + if 'BASE_3_14=YES' in myfile.read(): + makeargs = [] + + bindir = os.path.join(os.getcwd(), 'bin', os.environ['EPICS_HOST_ARCH']) + if os.path.isdir(bindir): + dllpaths.append(bindir) + + os.environ['PATH'] = os.pathsep.join(dllpaths + [os.environ['PATH']]) + def prepare(args): host_info() From 81bf29b8caef4ad3ff0f76d6e58971b6474f69f4 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Wed, 1 Apr 2020 15:34:13 +0200 Subject: [PATCH 52/68] appveyor: fix modlist() to allow MODULES in setup files --- appveyor/do.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/appveyor/do.py b/appveyor/do.py index 4a25e9c..86312c2 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -41,9 +41,12 @@ def modlist(): - add_modules = os.environ.get('ADD_MODULES', '').upper().split() - modules = os.environ.get('MODULES', '').upper().split() - ret = ['BASE'] + add_modules + modules + for var in ['ADD_MODULES', 'MODULES']: + setup.setdefault(var, '') + if var in os.environ: + setup[var] = os.environ[var] + logger.debug('ENV assignment: %s = %s', var, setup[var]) + ret = ['BASE'] + setup['ADD_MODULES'].upper().split() + setup['MODULES'].upper().split() logger.debug('Effective module list: %s', ret) return ret From 4a6a979f89a459ba6a0a244c1ccbc29e18544fcd Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Wed, 1 Apr 2020 18:14:46 +0200 Subject: [PATCH 53/68] appveyor: choco install re2c (for the sequencer) --- .appveyor.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.appveyor.yml b/.appveyor.yml index 047555d..53c72d9 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -15,6 +15,14 @@ cache: - C:\Users\appveyor\.tools -> appveyor\do.py +#---------------------------------# +# additional packages # +#---------------------------------# + +install: +# for the sequencer + - cinst re2c + #---------------------------------# # repository cloning # #---------------------------------# From d02dda5775525d0aa6d8ac404fb7f2132d5bc8e1 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Wed, 1 Apr 2020 18:57:21 +0200 Subject: [PATCH 54/68] appveyor: add keys to call_make(); make test-results --- appveyor/do.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/appveyor/do.py b/appveyor/do.py index 86312c2..5d27b23 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -52,7 +52,7 @@ def modlist(): zip7 = r'C:\Program Files\7-Zip\7z' make = '' -makeargs = [] +isbase314 = False def host_info(): print('{0}Python setup{1}'.format(ANSI_CYAN, ANSI_RESET)) @@ -189,10 +189,16 @@ def call_git(args, **kws): return exitcode def call_make(args=[], **kws): - if 'cwd' in kws: - place = kws['cwd'] + place = kws.get('cwd', os.getcwd()) + parallel = kws.pop('parallel', 2) + silent = kws.pop('silent', False) + # no parallel make for Base 3.14 + if parallel <= 0 or isbase314: + makeargs = [] else: - place = os.getcwd() + makeargs = ['-j{0}'.format(parallel), '-Otarget'] + if silent: + makeargs += ['-s'] logger.debug("EXEC '%s' in %s", ' '.join([make] + makeargs + args), place) sys.stdout.flush() exitcode = sp.call([make] + makeargs + args, **kws) @@ -315,7 +321,7 @@ def add_dependency(dep): update_release_local(setup[dep+"_VARNAME"], place) def setup_for_build(): - global make, makeargs + global make, isbase314 dllpaths = [] # there is no combined static and debug EPICS_HOST_ARCH target, @@ -354,7 +360,6 @@ def setup_for_build(): os.environ['PATH']]) make = os.path.join(toolsdir, 'make.exe') - makeargs = ['-j2', '-Otarget'] with open(os.path.join(cachedir, 'RELEASE.local'), 'r') as f: lines = f.readlines() @@ -365,10 +370,9 @@ def setup_for_build(): dllpaths.append(bindir) if mod == 'EPICS_BASE': base_place = place - # no parallel make for Base 3.14 with open(os.path.join(base_place, 'configure', 'CONFIG_BASE_VERSION')) as myfile: if 'BASE_3_14=YES' in myfile.read(): - makeargs = [] + isbase314 = True bindir = os.path.join(os.getcwd(), 'bin', os.environ['EPICS_HOST_ARCH']) if os.path.isdir(bindir): @@ -451,10 +455,9 @@ def prepare(args): setup_for_build() print('{0}EPICS_HOST_ARCH = {1}{2}'.format(ANSI_CYAN, os.environ['EPICS_HOST_ARCH'], ANSI_RESET)) - print('{0}make arguments = {1}{2}'.format(ANSI_CYAN, ' '.join(makeargs), ANSI_RESET)) print('{0}$ {1} --version{2}'.format(ANSI_CYAN, make, ANSI_RESET)) sys.stdout.flush() - sp.check_call([make, '--version']) + call_make(['--version'], parallel=0) print('{0}$ perl --version{1}'.format(ANSI_CYAN, ANSI_RESET)) sys.stdout.flush() sp.check_call(['perl', '--version']) @@ -471,7 +474,7 @@ def prepare(args): for mod in modlist(): place = places[setup[mod+"_VARNAME"]] print('{0}Building dependency {1} in {2}{3}'.format(ANSI_YELLOW, mod, place, ANSI_RESET)) - call_make(cwd=place) + call_make(cwd=place) #TBD: default should be silent def build(args): setup_for_build() @@ -481,7 +484,8 @@ def build(args): def test(args): setup_for_build() print('{0}Running the main module tests{1}'.format(ANSI_YELLOW, ANSI_RESET)) - call_make(['tapfiles']) + call_make(['tapfiles'], parallel=0) + call_make(['test-results'], parallel=0, silent=True) def doExec(args): 'exec user command with vcvars' From e5af5c2bfee540dc346b03b206d6d1da3a181235 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Thu, 2 Apr 2020 14:40:34 +0200 Subject: [PATCH 55/68] appveyor: print dependency table and RELEASE.local --- appveyor/do.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/appveyor/do.py b/appveyor/do.py index 5d27b23..8bb1164 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -476,6 +476,17 @@ def prepare(args): print('{0}Building dependency {1} in {2}{3}'.format(ANSI_YELLOW, mod, place, ANSI_RESET)) call_make(cwd=place) #TBD: default should be silent + print('{0}Dependency module information{1}'.format(ANSI_CYAN, ANSI_RESET)) + print('Module Tag Binaries Commit') + print(100 * '-') + for mod in modlist(): + commit = sp.check_output(['git', 'log', '-n1', '--oneline'], cwd=places[setup[mod+"_VARNAME"]]).strip() + print("%-10s %-12s %-11s %s" % (mod, setup[mod], 'rebuilt', commit)) + + print('{0}Contents of RELEASE.local{1}'.format(ANSI_CYAN, ANSI_RESET)) + with open(os.path.join(cachedir, 'RELEASE.local'), 'r') as f: + print(f.read().strip()) + def build(args): setup_for_build() print('{0}Building the main module{1}'.format(ANSI_YELLOW, ANSI_RESET)) From ddf6b961b106af56cffbc464fe11b5b595f8eba6 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Thu, 2 Apr 2020 12:01:26 +0200 Subject: [PATCH 56/68] appveyor: use VV to set logging level and silence dependency builds --- .appveyor.yml | 2 ++ appveyor/do.py | 9 +++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 53c72d9..75b638b 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -61,6 +61,7 @@ environment: SETUP_PATH: .:.ci SET: test01 BASE: 3.15 + VV: 1 matrix: - CC: vs2019 @@ -68,6 +69,7 @@ environment: SET: test00 - CC: mingw APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 + VV: 0 - CC: vs2019 APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 - CC: vs2017 diff --git a/appveyor/do.py b/appveyor/do.py index 8bb1164..9c19f6f 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -12,7 +12,6 @@ import distutils.util logger = logging.getLogger(__name__) -logging.basicConfig(level=logging.DEBUG) # Setup ANSI Colors ANSI_RED = "\033[31;1m" @@ -53,6 +52,7 @@ def modlist(): zip7 = r'C:\Program Files\7-Zip\7z' make = '' isbase314 = False +silent_dep_builds = True def host_info(): print('{0}Python setup{1}'.format(ANSI_CYAN, ANSI_RESET)) @@ -474,7 +474,7 @@ def prepare(args): for mod in modlist(): place = places[setup[mod+"_VARNAME"]] print('{0}Building dependency {1} in {2}{3}'.format(ANSI_YELLOW, mod, place, ANSI_RESET)) - call_make(cwd=place) #TBD: default should be silent + call_make(cwd=place, silent=silent_dep_builds) print('{0}Dependency module information{1}'.format(ANSI_CYAN, ANSI_RESET)) print('Module Tag Binaries Commit') @@ -580,7 +580,12 @@ def getargs(): return P def main(raw): + global silent_dep_builds args = getargs().parse_args(raw) + if 'VV' in os.environ and os.environ['VV'] == '1': + logging.basicConfig(level=logging.DEBUG) + silent_dep_builds = False + if args.vcvars and os.environ['CC'].startswith('vs'): # re-exec with MSVC in PATH with_vcvars(' '.join(['--no-vcvars']+raw)) From ca74a8424b8113ec0bb5a552338aea1a9d654678 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Thu, 2 Apr 2020 15:11:26 +0200 Subject: [PATCH 57/68] appveyor: create jobs for base 7.0, 3.15, 3.14 - default Base 7.0 / recursive = no - add base 3.15 and 3.14 on vs2019 --- .appveyor.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index 75b638b..9dee87c 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -60,7 +60,7 @@ environment: # common variables SETUP_PATH: .:.ci SET: test01 - BASE: 3.15 + BASE_RECURSIVE: NO VV: 1 matrix: @@ -69,8 +69,14 @@ environment: SET: test00 - CC: mingw APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 + - CC: vs2019 + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 VV: 0 - CC: vs2019 + BASE: 3.15 + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 + - CC: vs2019 + BASE: 3.14 APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 - CC: vs2017 APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 From 2fd1f9ec16423ffc88dd84843c7da5f1f260bb98 Mon Sep 17 00:00:00 2001 From: Michael Davidsaver Date: Wed, 1 Apr 2020 18:01:17 -0700 Subject: [PATCH 58/68] appveyor: show test-results after .tap upload --- appveyor/.appveyor.yml.example-full | 1 + 1 file changed, 1 insertion(+) diff --git a/appveyor/.appveyor.yml.example-full b/appveyor/.appveyor.yml.example-full index 662edd2..71acb1c 100644 --- a/appveyor/.appveyor.yml.example-full +++ b/appveyor/.appveyor.yml.example-full @@ -86,6 +86,7 @@ test_script: on_finish: - ps: Get-ChildItem *.tap -Recurse -Force | % { Push-AppveyorArtifact $_.FullName -FileName $_.Name } + - cmd: python .ci/appveyor/do.py build test-results -s #---------------------------------# # debugging # From 6d0f34ac653540e5314061f416acb3b616fb3412 Mon Sep 17 00:00:00 2001 From: Michael Davidsaver Date: Wed, 1 Apr 2020 18:05:54 -0700 Subject: [PATCH 59/68] appveyor: do --add-path --- appveyor/do.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/appveyor/do.py b/appveyor/do.py index 9c19f6f..bd712a2 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -320,7 +320,7 @@ def add_dependency(dep): update_release_local(setup[dep+"_VARNAME"], place) -def setup_for_build(): +def setup_for_build(args): global make, isbase314 dllpaths = [] @@ -380,6 +380,20 @@ def setup_for_build(): os.environ['PATH'] = os.pathsep.join(dllpaths + [os.environ['PATH']]) + # apparently %CD% is handled automagically + os.environ['TOP'] = os.getcwd() + + addpaths = [] + for path in args.paths: + try: + addpaths.append(path.format(**os.environ)) + except KeyError: + print('Environment') + [print(' ',K,'=',repr(V)) for K,V in os.environ.items()] + raise + + os.environ['PATH'] = os.pathsep.join([os.environ['PATH']] + addpaths) + def prepare(args): host_info() @@ -452,7 +466,7 @@ def prepare(args): sp.check_call('relocation.pl.bat', shell=True, stdout=devnull, cwd=os.path.join(toolsdir, 'strawberry')) - setup_for_build() + setup_for_build(args) print('{0}EPICS_HOST_ARCH = {1}{2}'.format(ANSI_CYAN, os.environ['EPICS_HOST_ARCH'], ANSI_RESET)) print('{0}$ {1} --version{2}'.format(ANSI_CYAN, make, ANSI_RESET)) @@ -488,19 +502,19 @@ def prepare(args): print(f.read().strip()) def build(args): - setup_for_build() + setup_for_build(args) print('{0}Building the main module{1}'.format(ANSI_YELLOW, ANSI_RESET)) call_make(args.makeargs) def test(args): - setup_for_build() + setup_for_build(args) print('{0}Running the main module tests{1}'.format(ANSI_YELLOW, ANSI_RESET)) call_make(['tapfiles'], parallel=0) call_make(['test-results'], parallel=0, silent=True) def doExec(args): 'exec user command with vcvars' - setup_for_build() + setup_for_build(args) os.environ['MAKE'] = make print('Execute command {}'.format(args.cmd)) sys.stdout.flush() @@ -561,6 +575,8 @@ def getargs(): P = ArgumentParser() P.add_argument('--no-vcvars', dest='vcvars', default=True, action='store_false', help='Assume vcvarsall.bat has already been run') + P.add_argument('--add-path', dest='paths', default=[], action='append', + help='Append directory to %PATH%. Expands {ENVVAR}') SP = P.add_subparsers() CMD = SP.add_parser('prepare') From 16bb305d24efc0e5a34588f6f5d3918c34c26c82 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Fri, 3 Apr 2020 14:59:04 +0200 Subject: [PATCH 60/68] appveyor: add setup_for_build() tests --- appveyor-test.py | 84 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/appveyor-test.py b/appveyor-test.py index 76fbdec..d94e13b 100644 --- a/appveyor-test.py +++ b/appveyor-test.py @@ -12,6 +12,8 @@ import re import subprocess as sp import unittest +import logging +from argparse import Namespace builddir = os.getcwd() @@ -294,7 +296,89 @@ def test_vcvars(self): do.with_vcvars('env') +class TestSetupForBuild(unittest.TestCase): + configuration = os.environ['CONFIGURATION'] + platform = os.environ['PLATFORM'] + cc = os.environ['CC'] + args = Namespace(paths=[]) + + def setUp(self): + os.environ.pop('EPICS_HOST_ARCH', None) + + def tearDown(self): + os.environ['CONFIGURATION'] = self.configuration + os.environ['PLATFORM'] = self.platform + os.environ['CC'] = self.cc + + def test_AddPathsOption(self): + os.environ['FOOBAR'] = 'BAR' + args = Namespace(paths=['/my/{FOOBAR}/dir', '/my/foobar']) + do.setup_for_build(args) + self.assertTrue(re.search('/my/BAR/dir', os.environ['PATH']), 'Expanded path not in PATH') + self.assertTrue(re.search('/foobar', os.environ['PATH']), 'Plain path not in PATH') + os.environ.pop('FOOBAR', None) + + def test_HostArchConfiguration(self): + for config in ['dynamic', 'dynamic-debug', 'static', 'static-debug']: + os.environ['CONFIGURATION'] = config + do.setup_for_build(self.args) + self.assertTrue('EPICS_HOST_ARCH' in os.environ, + 'EPICS_HOST_ARCH is not set for Configuration={0}'.format(config)) + if re.search('static', config): + self.assertTrue(re.search('-static$', os.environ['EPICS_HOST_ARCH']), + 'EPICS_HOST_ARCH is not -static for Configuration={0}'.format(config)) + self.assertFalse(re.search('debug', os.environ['EPICS_HOST_ARCH']), + 'EPICS_HOST_ARCH is -debug for Configuration={0}'.format(config)) + elif re.search('debug', config): + self.assertFalse(re.search('static', os.environ['EPICS_HOST_ARCH']), + 'EPICS_HOST_ARCH is -static for Configuration={0}'.format(config)) + self.assertTrue(re.search('-debug$', os.environ['EPICS_HOST_ARCH']), + 'EPICS_HOST_ARCH is not -debug for Configuration={0}'.format(config)) + else: + self.assertFalse(re.search('static', os.environ['EPICS_HOST_ARCH']), + 'EPICS_HOST_ARCH is -static for Configuration={0}'.format(config)) + self.assertFalse(re.search('debug', os.environ['EPICS_HOST_ARCH']), + 'EPICS_HOST_ARCH is -debug for Configuration={0}'.format(config)) + + def test_HostArchPlatform(self): + for platform in ['x86', 'x64', 'X64']: + for cc in ['vs2019', 'mingw']: + os.environ['PLATFORM'] = platform + os.environ['CC'] = cc + os.environ['CONFIGURATION'] = 'dynamic' + do.setup_for_build(self.args) + self.assertTrue('EPICS_HOST_ARCH' in os.environ, + 'EPICS_HOST_ARCH is not set for {0} / {1}'.format(cc, platform)) + if platform == 'x86': + self.assertTrue(re.search('^win32-x86', os.environ['EPICS_HOST_ARCH']), + 'EPICS_HOST_ARCH is not win32-x86 for {0} / {1}'.format(cc, platform)) + else: + self.assertTrue(re.search('^windows-x64', os.environ['EPICS_HOST_ARCH']), + 'EPICS_HOST_ARCH is not windows-x64 for {0} / {1}'.format(cc, platform)) + if cc == 'mingw': + self.assertTrue(re.search('-mingw$', os.environ['EPICS_HOST_ARCH']), + 'EPICS_HOST_ARCH is not -mingw for {0} / {1}'.format(cc, platform)) + if platform == 'x86': + pattern = 'mingw32' + else: + pattern = 'mingw64' + self.assertTrue(re.search(pattern, os.environ['PATH']), + 'Binary location for {0} not in PATH'.format(pattern)) + self.assertTrue(re.search(pattern, os.environ['INCLUDE']), + 'Include location for {0} not in INCLUDE'.format(pattern)) + + def test_StrawberryInPath(self): + os.environ['CC'] = 'vs2019' + do.setup_for_build(self.args) + self.assertTrue(re.search('strawberry', os.environ['PATH']), + 'Strawberry Perl location not in PATH for vs2019') + + if __name__ == "__main__": + if 'VV' in os.environ and os.environ['VV'] == '1': + logging.basicConfig(level=logging.DEBUG) + do.silent_dep_builds = False + do.host_info() if sys.argv[1:]==['env']: # testing with_vcvars From d8e53e84fddb8bdff2d2ae8a298fc622920128f7 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Fri, 3 Apr 2020 18:25:55 +0200 Subject: [PATCH 61/68] appveyor: CC -> CMP for compiler toolchain setting --- .appveyor.yml | 33 ++++++++++++++++----------------- appveyor-test.py | 10 +++++----- appveyor/do.py | 12 ++++++------ 3 files changed, 27 insertions(+), 28 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 9dee87c..4bfc45c 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -55,36 +55,35 @@ configuration: - dynamic-debug - static-debug -# Environment variables: compiler toolchain +# Environment variables: compiler toolchain, base version, setup file, ... environment: - # common variables + # common / default variables for all jobs SETUP_PATH: .:.ci SET: test01 BASE_RECURSIVE: NO VV: 1 matrix: - - CC: vs2019 + - CMP: vs2019 APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 SET: test00 - - CC: mingw - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 - - CC: vs2019 + - CMP: mingw + - CMP: vs2019 APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 VV: 0 - - CC: vs2019 + - CMP: vs2019 BASE: 3.15 APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 - - CC: vs2019 + - CMP: vs2019 BASE: 3.14 APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 - - CC: vs2017 + - CMP: vs2017 APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 - - CC: vs2015 - - CC: vs2013 - - CC: vs2012 - - CC: vs2010 - - CC: vs2008 + - CMP: vs2015 + - CMP: vs2013 + - CMP: vs2012 + - CMP: vs2010 + - CMP: vs2008 # Platform: architecture platform: @@ -105,11 +104,11 @@ matrix: SET: test00 # VS2012 and older installs don't have the 64 bit compiler - platform: x64 - CC: vs2012 + CMP: vs2012 - platform: x64 - CC: vs2010 + CMP: vs2010 - platform: x64 - CC: vs2008 + CMP: vs2008 # Run test script for unit tests (SET = test00) for: diff --git a/appveyor-test.py b/appveyor-test.py index d94e13b..b2bcc4d 100644 --- a/appveyor-test.py +++ b/appveyor-test.py @@ -290,7 +290,7 @@ def test_Repos(self): class TestVCVars(unittest.TestCase): def test_vcvars(self): - if ('CC' in os.environ and os.environ['CC'] in ('mingw',)) \ + if ('CMP' in os.environ and os.environ['CMP'] in ('mingw',)) \ or distutils.util.get_platform() != "win32": raise unittest.SkipTest() @@ -299,7 +299,7 @@ def test_vcvars(self): class TestSetupForBuild(unittest.TestCase): configuration = os.environ['CONFIGURATION'] platform = os.environ['PLATFORM'] - cc = os.environ['CC'] + cc = os.environ['CMP'] args = Namespace(paths=[]) def setUp(self): @@ -308,7 +308,7 @@ def setUp(self): def tearDown(self): os.environ['CONFIGURATION'] = self.configuration os.environ['PLATFORM'] = self.platform - os.environ['CC'] = self.cc + os.environ['CMP'] = self.cc def test_AddPathsOption(self): os.environ['FOOBAR'] = 'BAR' @@ -344,7 +344,7 @@ def test_HostArchPlatform(self): for platform in ['x86', 'x64', 'X64']: for cc in ['vs2019', 'mingw']: os.environ['PLATFORM'] = platform - os.environ['CC'] = cc + os.environ['CMP'] = cc os.environ['CONFIGURATION'] = 'dynamic' do.setup_for_build(self.args) self.assertTrue('EPICS_HOST_ARCH' in os.environ, @@ -368,7 +368,7 @@ def test_HostArchPlatform(self): 'Include location for {0} not in INCLUDE'.format(pattern)) def test_StrawberryInPath(self): - os.environ['CC'] = 'vs2019' + os.environ['CMP'] = 'vs2019' do.setup_for_build(self.args) self.assertTrue(re.search('strawberry', os.environ['PATH']), 'Strawberry Perl location not in PATH for vs2019') diff --git a/appveyor/do.py b/appveyor/do.py index bd712a2..cae41ec 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -338,12 +338,12 @@ def setup_for_build(args): elif os.environ['PLATFORM'].lower() == 'x64': os.environ['EPICS_HOST_ARCH'] = 'windows-x64' + hostarchsuffix - if os.environ['CC'] == 'vs2019': + if os.environ['CMP'] == 'vs2019': # put our strawberry 'perl' in the PATH os.environ['PATH'] = os.pathsep.join([os.path.join(toolsdir, 'strawberry', 'perl', 'site', 'bin'), os.path.join(toolsdir, 'strawberry', 'perl', 'bin'), os.environ['PATH']]) - if os.environ['CC'] == 'mingw': + if os.environ['CMP'] == 'mingw': if 'INCLUDE' not in os.environ: os.environ['INCLUDE'] = '' if os.environ['PLATFORM'].lower() == 'x86': @@ -453,7 +453,7 @@ def prepare(args): os.remove(os.path.join(toolsdir, 'make-{0}.zip'.format(makever))) perlver = '5.30.0.1' - if os.environ['CC'] == 'vs2019': + if os.environ['CMP'] == 'vs2019': if not os.path.isdir(os.path.join(toolsdir, 'strawberry')): print('Installing Strawberry Perl {0}'.format(perlver)) sys.stdout.flush() @@ -476,7 +476,7 @@ def prepare(args): sys.stdout.flush() sp.check_call(['perl', '--version']) - if os.environ['CC'] == 'mingw': + if os.environ['CMP'] == 'mingw': print('{0}$ gcc --version{1}'.format(ANSI_CYAN, ANSI_RESET)) sys.stdout.flush() sp.check_call(['gcc', '--version']) @@ -523,7 +523,7 @@ def doExec(args): def with_vcvars(cmd): '''re-exec main script with a (hopefully different) command ''' - CC = os.environ['CC'] + CC = os.environ['CMP'] # cf. https://docs.microsoft.com/en-us/cpp/build/building-on-the-command-line @@ -602,7 +602,7 @@ def main(raw): logging.basicConfig(level=logging.DEBUG) silent_dep_builds = False - if args.vcvars and os.environ['CC'].startswith('vs'): + if args.vcvars and os.environ['CMP'].startswith('vs'): # re-exec with MSVC in PATH with_vcvars(' '.join(['--no-vcvars']+raw)) From b502aa7049bb222262ebe407611bc9e3128ddf3f Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Fri, 3 Apr 2020 16:12:11 +0200 Subject: [PATCH 62/68] appveyor: update .appveyor.yml.example-full --- appveyor/.appveyor.yml.example-full | 78 +++++++++++++++++++++-------- 1 file changed, 56 insertions(+), 22 deletions(-) diff --git a/appveyor/.appveyor.yml.example-full b/appveyor/.appveyor.yml.example-full index 71acb1c..5288650 100644 --- a/appveyor/.appveyor.yml.example-full +++ b/appveyor/.appveyor.yml.example-full @@ -3,6 +3,23 @@ # This is YAML - indentation levels are crucial +#---------------------------------# +# build cache # +#---------------------------------# +# The AppVeyor cache allowance is way too small (1GB per account across all projects, branches and jobs) +# to be used for the dependency builds. + +cache: + - C:\Users\appveyor\.tools -> appveyor\do.py + +#---------------------------------# +# additional packages # +#---------------------------------# + +install: +# for the sequencer + - cinst re2c + #---------------------------------# # repository cloning # #---------------------------------# @@ -27,6 +44,10 @@ skip_commits: # build matrix configuration # #---------------------------------# +# Since dependencies cannot be cached and AppVeyor only grants a single builder VM, all jobs +# are executed sequentially, each one taking 10-15 minutes. +# Consider this when defining your build matrix. (A full matrix build takes more than 8 hours.) + # Build Configurations: dll/static, regular/debug configuration: - dynamic @@ -34,41 +55,54 @@ configuration: - dynamic-debug - static-debug -# Environment variables: compiler toolchain +# Environment variables: compiler toolchain, base version, setup file, ... environment: + # common / default variables for all jobs + SETUP_PATH: .ci-local:.ci + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 + matrix: - - CC: mingw - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 - - CC: vs2019 + - CMP: vs2019 + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 + SET: test00 + - CMP: mingw + - CMP: vs2019 APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 - - CC: vs2017 + - CMP: vs2019 + BASE: 3.15 + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 + - CMP: vs2019 + BASE: 3.14 + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 + - CMP: vs2017 APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 - - CC: vs2015 - - CC: vs2013 - - CC: vs2012 - - CC: vs2010 + - CMP: vs2015 + - CMP: vs2013 + - CMP: vs2012 + - CMP: vs2010 + - CMP: vs2008 -# Platform: architecture +# Platform: processor architecture platform: - x86 - x64 -# Matrix configuration: allow specific failing jobs +# Matrix configuration: exclude sets of jobs matrix: exclude: - # VS2010 Express installs don't have the 64 bit compiler + # VS2012 and older installs don't have the 64 bit compiler + - platform: x64 + CMP: vs2012 - platform: x64 - CC: vs2010 - # Exclude to reduce total job runtime - # skip 64-bit for older and 32-bit for newer + CMP: vs2010 - platform: x64 - CC: vs2012 - - platform: x86 - CC: mingw - - platform: x86 - CC: vs2019 - - platform: x86 - CC: vs2017 + CMP: vs2008 + # Exclude more jobs to reduce build time + # E.g., skip 32-bit for newer compilers + #- platform: x86 + # CMP: vs2019 + #- platform: x86 + # CMP: vs2017 #---------------------------------# # building & testing # From a3532d3c5536284a65a75bab447e37760d01935d Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Tue, 7 Apr 2020 15:49:53 +0200 Subject: [PATCH 63/68] appveyor: make cachedir configurable --- appveyor/do.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/appveyor/do.py b/appveyor/do.py index cae41ec..7c2dc37 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -38,6 +38,8 @@ cachedir = os.path.join('.', '.cache') toolsdir = os.path.join('.', '.tools') +if 'CACHEDIR' in os.environ: + cachedir = os.environ['CACHEDIR'] def modlist(): for var in ['ADD_MODULES', 'MODULES']: From 22d3a9db15425412f97f8f3ce4d52b8d76cd0cf2 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Wed, 8 Apr 2020 14:43:45 +0200 Subject: [PATCH 64/68] appveyor: add MSI 1.7 to any Base 3.14 dependency - add test for patch being applied --- appveyor-test.py | 11 +++++++++-- appveyor/do.py | 18 ++++++++++++++++-- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/appveyor-test.py b/appveyor-test.py index b2bcc4d..8d3001d 100644 --- a/appveyor-test.py +++ b/appveyor-test.py @@ -228,8 +228,8 @@ class TestAddDependencyOptions(unittest.TestCase): def setUp(self): os.environ['SETUP_PATH'] = '.:appveyor' - if os.path.exists(self.location): - shutil.rmtree(self.location, onerror=do.remove_readonly) + if os.path.exists(do.cachedir): + shutil.rmtree(do.cachedir, onerror=do.remove_readonly) do.clear_lists() do.source_set('defaults') do.complete_setup('MCoreUtils') @@ -258,6 +258,13 @@ def test_SetDepthThree(self): self.assertTrue(is_shallow_repo(self.location), 'Module not checked out shallow (requested: default=5)') + def test_AddMsiTo314(self): + do.complete_setup('BASE') + do.setup['BASE'] = 'R3.14.12.1' + msifile = os.path.join(do.cachedir, 'base-R3.14.12.1', 'src', 'dbtools', 'msi.c') + do.add_dependency('BASE') + self.assertTrue(os.path.exists(msifile), 'MSI was not added to Base 3.14') + def repo_access(dep): do.set_setup_from_env(dep) do.setup.setdefault(dep + "_DIRNAME", dep.lower()) diff --git a/appveyor/do.py b/appveyor/do.py index 7c2dc37..d4d4e35 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -41,6 +41,10 @@ if 'CACHEDIR' in os.environ: cachedir = os.environ['CACHEDIR'] +ciscriptsdir = os.path.abspath(os.path.dirname(sys.argv[0])) +if os.path.basename(ciscriptsdir) == 'appveyor': + ciscriptsdir = ciscriptsdir.rstrip(os.pathsep+'appveyor') + def modlist(): for var in ['ADD_MODULES', 'MODULES']: setup.setdefault(var, '') @@ -298,8 +302,18 @@ def add_dependency(dep): sp.check_call(['git', 'log', '-n1'], cwd=place) modules_to_compile.append(place) - # force including RELEASE.local for non-base modules by overwriting their configure/RELEASE - if dep != 'BASE': + if dep == 'BASE': + # add MSI 1.7 to Base 3.14 + versionfile = os.path.join(place, 'configure', 'CONFIG_BASE_VERSION') + if os.path.exists(versionfile): + with open(versionfile) as f: + if 'BASE_3_14=YES' in f.read(): + print('Adding MSI 1.7 to {0}'.format(place)) + sys.stdout.flush() + sp.check_call(['patch', '-p0', '-i', os.path.join(ciscriptsdir, 'add-msi-to-314.patch')], + cwd=place) + else: + # force including RELEASE.local for non-base modules by overwriting their configure/RELEASE release = os.path.join(place, "configure", "RELEASE") if os.path.exists(release): with open(release, 'w') as fout: From 11e0a60e3b6975470af97b4d408da0e85c4bc2bb Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Wed, 8 Apr 2020 15:56:00 +0200 Subject: [PATCH 65/68] appveyor: update example and README files --- README.md | 20 +++++++---- appveyor/.appveyor.yml.example-full | 8 ++--- appveyor/README.md | 53 +++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 12 deletions(-) create mode 100644 appveyor/README.md diff --git a/README.md b/README.md index bcda044..ed7071f 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ ![Version][badge.version] ![Travis status][badge.travis] +![AppVeyor status][badge.appveyor] # Continuous Integration Scripts for EPICS Modules @@ -56,11 +57,17 @@ example. - Compile on MacOS - Built dependencies are cached (for faster builds) +### AppVeyor + - Use different compilers (Visual Studio, MinGW) + - Use different Visual Studio versions: \ + 2008, 2010, 2012, 2013, 2015, 2017, 2019 + - Compile for Windows 32bit and 64bit + ## How to Use the CI-Scripts 1. Get an account on a supported CI service provider platform. (e.g. [Travis-CI](https://travis-ci.org/), - AppVeyor, Azure Pipelines...) + [AppVeyor](https://www.appveyor.com/), Azure Pipelines...) (More details in the specific README of the subdirectory.) @@ -79,10 +86,10 @@ example. BASE=3.15 ASYN=R4-34 - SNCSEQ=R2-2-7 + SNCSEQ=R2-2-8 ``` will compile against the EPICS Base 3.15 branch, the Sequencer - release 2.2.7 and release 4.34 of asyn. + release 2.2.8 and release 4.34 of asyn. (Any settings can be overridden from the specific job configuration in e.g. `.travis.yml`.) @@ -214,16 +221,16 @@ This will make all builds (not just for your module) verbose. Update the submodule in `.ci` first, then change your CI configuration (if needed) and commit both to your module. E.g., to update your Travis -setup to release 2.1.0 of ci-scripts: +setup to release 2.2.1 of ci-scripts: ```bash cd .ci -git pull origin v2.1.0 +git pull origin v2.2.1 cd - git add .ci # if needed: edit .travis.yml git add .travis.yml -git commit -m "Update ci-scripts submodule to v2.1.0" +git commit -m "Update ci-scripts submodule to v2.2.1" ``` Check the example configuration files inside ci-scripts (and their @@ -266,6 +273,7 @@ in file LICENSE that is included with this distribution. [badge.version]: https://badge.fury.io/gh/epics-base%2Fci-scripts.svg [badge.travis]: https://travis-ci.org/epics-base/ci-scripts.svg?branch=master +[badge.appveyor]: https://ci.appveyor.com/api/projects/status/xwdv8fpxu0byp3hn?svg=true [reddit.bash]: https://www.reddit.com/r/bash/comments/393oqv/why_is_the_version_of_bash_included_in_os_x_so_old/ diff --git a/appveyor/.appveyor.yml.example-full b/appveyor/.appveyor.yml.example-full index 5288650..ee1f0da 100644 --- a/appveyor/.appveyor.yml.example-full +++ b/appveyor/.appveyor.yml.example-full @@ -10,7 +10,7 @@ # to be used for the dependency builds. cache: - - C:\Users\appveyor\.tools -> appveyor\do.py + - C:\Users\appveyor\.tools #---------------------------------# # additional packages # @@ -59,7 +59,6 @@ configuration: environment: # common / default variables for all jobs SETUP_PATH: .ci-local:.ci - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 matrix: - CMP: vs2019 @@ -108,11 +107,8 @@ matrix: # building & testing # #---------------------------------# -install: - - cmd: git submodule update --init --recursive - - cmd: python .ci/appveyor/do.py prepare - build_script: + - cmd: python .ci/appveyor/do.py prepare - cmd: python .ci/appveyor/do.py build test_script: diff --git a/appveyor/README.md b/appveyor/README.md new file mode 100644 index 0000000..d7e1175 --- /dev/null +++ b/appveyor/README.md @@ -0,0 +1,53 @@ +# AppVeyor Scripts for EPICS Modules + +## Features + + - Use different compilers (Visual Studio, MinGW) + - Use different VS versions (2008, 2010, 2012, 2013, 2015, 2017, 2019) + - Compile for Windows 32bit and 64bit + - Create static libraries or DLLs (plus the matching executables) + - Create optimized or debug builds + +## How to Use these Scripts + + 1. Get an account on [AppVeyor](https://www.appveyor.com/), connect + it to your GitHub account and activate your support module's + repository. For more details, please refer to the + [AppVeyor documentation](https://www.appveyor.com/docs/). + + 2. Add the ci-scripts respository as a Git Submodule + (see [README](../README.md) one level above). + + 3. Add settings files defining which dependencies in which versions + you want to build against + (see [README](../README.md) one level above). + + 4. Create an AppVeyor configuration by copying one of the examples into + the root directory of your module. + ``` + $ cp .ci/appveyor/.appveyor.yml.example-full .appveyor.yml + ``` + + 5. Edit the `.appveyor.yml` configuration to include the jobs you want + AppVeyor to run. + + AppVeyor automatically creates a build matrix with the following axes: + 1. `configuration:` \ + Select static or dynamic (DLL) as well as regular or debug builds. + 2. `platform:` \ + Select 32bit or 64bit processor architecture. + 3. `environment: / matrix:` \ + List of environment variable settings. Each list element (starting with + a dash) is one step on the axis of the build matrix. + + Your builds will take long. + + AppVeyor only grants a single worker VM - all jobs of the matrix are + executed sequentially. Each job will take around 10 minutes. + + The `matrix: / exclude:` setting can be used to reduce the number of + jobs. Check the [AppVeyor docs](https://www.appveyor.com/docs/build-configuration/#build-matrix) + for more ways to reduce the build matrix size. + + 6. Push your changes and check + [ci.appveyor.com](https://ci.appveyor.com/) for your build results. From 070eab1473a0f11205edc73678285a548dd1b6e5 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Tue, 21 Apr 2020 17:31:24 +0200 Subject: [PATCH 66/68] appveyor: re-add recursive submodule update --- appveyor/.appveyor.yml.example-full | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/appveyor/.appveyor.yml.example-full b/appveyor/.appveyor.yml.example-full index ee1f0da..f8b57b5 100644 --- a/appveyor/.appveyor.yml.example-full +++ b/appveyor/.appveyor.yml.example-full @@ -107,8 +107,11 @@ matrix: # building & testing # #---------------------------------# -build_script: +install: + - cmd: git submodule update --init --recursive - cmd: python .ci/appveyor/do.py prepare + +build_script: - cmd: python .ci/appveyor/do.py build test_script: From 32bdf8480645d1bd9b78409ee9593950ec99ee8e Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Tue, 21 Apr 2020 17:40:29 +0200 Subject: [PATCH 67/68] appveyor: improve RDP debugging options --- appveyor/.appveyor.yml.example-full | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/appveyor/.appveyor.yml.example-full b/appveyor/.appveyor.yml.example-full index f8b57b5..d7f4972 100644 --- a/appveyor/.appveyor.yml.example-full +++ b/appveyor/.appveyor.yml.example-full @@ -129,8 +129,12 @@ on_finish: ## note that you will need to connect within the usual build timeout limit (60 minutes) ## so you may want to adjust the build matrix above to just build the one of interest -#on_failure: +# print the connection info +#init: # - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) + +# block a failed build (until the watchdog barks) +#on_failure: # - ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) #---------------------------------# From 79cc56059497a032ab9331777269a731ba58f74c Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Tue, 21 Apr 2020 17:48:05 +0200 Subject: [PATCH 68/68] appveyor: add minimal example file --- appveyor/.appveyor.yml.example-mini | 71 +++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 appveyor/.appveyor.yml.example-mini diff --git a/appveyor/.appveyor.yml.example-mini b/appveyor/.appveyor.yml.example-mini new file mode 100644 index 0000000..1293751 --- /dev/null +++ b/appveyor/.appveyor.yml.example-mini @@ -0,0 +1,71 @@ +# .appveyor.yml for use with EPICS Base ci-scripts +# (see: https://github.com/epics-base/ci-scripts) + +# This is YAML - indentation levels are crucial + +cache: + - C:\Users\appveyor\.tools + +init: + - git config --global core.autocrlf true + +clone_depth: 50 + +skip_commits: + files: + - 'documentation/*' + - 'templates/*' + - '**/*.html' + - '**/*.md' + +# Build Configurations: dll/static, regular/debug +configuration: + - dynamic +# - static + - dynamic-debug +# - static-debug + +environment: + # common / default variables for all jobs + SETUP_PATH: .ci-local:.ci + + matrix: + - CMP: vs2019 + BASE: 7.0 + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 + - CMP: vs2019 + BASE: 3.15 + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 + +# Platform: processor architecture +platform: +# - x86 + - x64 + +# Matrix configuration: exclude sets of jobs +matrix: + exclude: + # VS2012 and older installs don't have the 64 bit compiler + - platform: x64 + CMP: vs2012 + - platform: x64 + CMP: vs2010 + - platform: x64 + CMP: vs2008 + +install: + - cmd: git submodule update --init --recursive + - cmd: python .ci/appveyor/do.py prepare + +build_script: + - cmd: python .ci/appveyor/do.py build + +test_script: + - cmd: python .ci/appveyor/do.py test + +on_finish: + - ps: Get-ChildItem *.tap -Recurse -Force | % { Push-AppveyorArtifact $_.FullName -FileName $_.Name } + - cmd: python .ci/appveyor/do.py build test-results -s + +notifications: + - provider: GitHubPullRequest