diff --git a/.gitignore b/.gitignore index 781785621..4b98d286b 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,12 @@ # Build artifacts target/ +*.pyc +*.swp +.depproj +.vs # Drivers modules/c/j2k/external/openjpeg/openjpeg-2.0.0/ modules/c/j2k/external/jasper/jasper-1.900.1-mod/ + diff --git a/externals/coda-oss/.gitignore b/externals/coda-oss/.gitignore index 7f9dafe40..e42feafad 100644 --- a/externals/coda-oss/.gitignore +++ b/externals/coda-oss/.gitignore @@ -4,6 +4,7 @@ # Build artifacts +install/ target/ # Waf diff --git a/externals/coda-oss/build/build.py b/externals/coda-oss/build/build.py index 81418cf9c..a74795959 100644 --- a/externals/coda-oss/build/build.py +++ b/externals/coda-oss/build/build.py @@ -276,15 +276,21 @@ def module(self, **modArgs): lang=lang, path=testNode, includes=includes, defines=defines, install_path='${PREFIX}/tests/%s' % modArgs['name']) - pythonTestNode = path.parent.parent.make_node('python').make_node(str(path)).make_node('tests') - if os.path.exists(pythonTestNode.abspath()) and not Options.options.libs_only: - tests = [str(test) for test in pythonTestNode.ant_glob('*.py') if - str(test) not in listify(modArgs.get('test_filter', ''))] - for test in tests: - bld(features='install_tgt', - files=[test], dir=pythonTestNode, - name=test, target=test, - install_path='${PREFIX}/tests/%s' % modArgs['name']) + + # Create install target for python tests + if not Options.options.libs_only: + for testDirname in ['tests', 'unittests']: + pythonTestNode = path.parent.parent.make_node('python').\ + make_node(str(path)).make_node(testDirname) + if os.path.exists(pythonTestNode.abspath()): + tests = [str(test) for test in pythonTestNode.ant_glob('*.py') if + str(test) not in listify(modArgs.get('test_filter', ''))] + for test in tests: + installPath = '${PREFIX}/%s/%s' % (testDirname, modArgs['name']) + bld(features='install_tgt', + files=[test], dir=pythonTestNode, + name=test, target=test, + install_path=installPath) testNode = path.make_node('unittests') @@ -305,24 +311,14 @@ def module(self, **modArgs): for test in testNode.ant_glob('*%s' % sourceExtensions): if str(test) not in listify(modArgs.get('unittest_filter', '')): testName = splitext(str(test))[0] - exe = self(features='%s %sprogram' % (libExeType, libExeType), + exe = self(features='%s %sprogram test' % (libExeType, libExeType), env=env.derive(), name=testName, target=testName, source=str(test), use=test_deps, uselib = modArgs.get('unittest_uselib', modArgs.get('uselib', '')), lang=lang, path=testNode, defines=defines, includes=includes, install_path='${PREFIX}/unittests/%s' % modArgs['name']) - if Options.options.unittests or Options.options.all_tests: - exe.features += ' test' - tests.append(testName) - # add a post-build hook to run the unit tests - # I use partial so I can pass arguments to a post build hook - #if Options.options.unittests: - # bld.add_post_fun(partial(CPPBuildContext.runUnitTests, - # tests=tests, - # path=self.getBuildDir(testNode))) - confDir = path.make_node('conf') if exists(confDir.abspath()): lib.targets_to_add.append( @@ -339,7 +335,7 @@ def plugin(self, **modArgs): plugin (via the plugin kwarg). """ bld = self - env = self._getEnv(modArgs) + env = self._getEnv(modArgs).derive() modArgs = dict((k.lower(), v) for k, v in list(modArgs.items())) lang = modArgs.get('lang', 'c++') @@ -370,7 +366,7 @@ def plugin(self, **modArgs): lib = bld(features='%s %sshlib add_targets no_implib' % (libExeType, libExeType), target=libName, name=targetName, source=source, includes=includes, export_includes=exportIncludes, - use=uselib_local, uselib=uselib, env=env.derive(), + use=uselib_local, uselib=uselib, env=env, defines=defines, path=path, targets_to_add=targetsToAdd, install_path=join(env['install_sharedir'], plugin, 'plugins')) @@ -545,7 +541,7 @@ def swigModule(self, **modArgs): # actually check it in so other developers can still use the Python # bindings even if they don't have Swig flags = '-python -c++' - if sys.version_info[0] >= 3: + if sys.version_info[0] >= 3 and not env['PYTHON_AGNOSTIC']: flags += ' -py3' bld(features = 'cxx cshlib pyext add_targets swig_linkage includes', source = swigSource, @@ -643,7 +639,7 @@ def next(self): except IndexError: # pop next directory from stack if len(self.stack) == 0: - raise StopIteration + return self.directory = self.stack.pop() if isdir(self.directory): self.files = os.listdir(self.directory) @@ -724,6 +720,11 @@ def unzipper(inFile, outDir): outFile.flush() outFile.close() + +def deprecated_callback(option, opt, value, parser): + Logs.warn('Warning: {0} is deprecated'.format(opt)) + + def options(opt): opt.load('compiler_cc') opt.load('compiler_cxx') @@ -747,8 +748,7 @@ def options(opt): default=False, help='Treat compiler warnings as errors') opt.add_option('--enable-debugging', action='store_true', dest='debugging', help='Enable debugging') - opt.add_option('--enable-cpp11', action='store_true', default=False, dest='enablecpp11', - help='Enable C++11 features') + opt.add_option('--enable-cpp11', action='callback', callback=deprecated_callback) #TODO - get rid of enable64 - it's useless now opt.add_option('--enable-64bit', action='store_true', dest='enable64', help='Enable 64bit builds') @@ -772,8 +772,6 @@ def options(opt): help='Build all libs as shared libs') opt.add_option('--disable-symlinks', action='store_false', dest='symlinks', default=True, help='Disable creating symlinks for libs') - opt.add_option('--unittests', action='store_true', dest='unittests', - help='Build-time option to run unit tests after the build has completed') opt.add_option('--no-headers', action='store_false', dest='install_headers', default=True, help='Don\'t install module headers') opt.add_option('--no-libs', action='store_false', dest='install_libs', @@ -884,12 +882,7 @@ def configureCompilerOptions(self): config['cxx']['optz_fast'] = '-O2' config['cxx']['optz_fastest'] = '-O3' - gxxCompileFlags='-fPIC' - if self.env['cpp11support'] and \ - ((cxxCompiler == 'g++' and gccHasCpp11()) or \ - (cxxCompiler == 'icpc' and iccHasCpp11())): - gxxCompileFlags+=' -std=c++11' - + gxxCompileFlags='-fPIC -std=c++11' self.env.append_value('CXXFLAGS', gxxCompileFlags.split()) # DEFINES and LINKFLAGS will apply to both gcc and g++ @@ -1193,6 +1186,9 @@ def configure(self): if self.env['DETECTED_BUILD_PY']: return + if sys.version_info < (2, 7, 0): + self.fatal('build.py requires at least Python 2.7') + sys_platform = getPlatform(default=Options.platform) winRegex = r'win32' @@ -1281,14 +1277,17 @@ def configure(self): env.append_unique('LINKFLAGS', Options.options.linkflags.split()) if Options.options._defs: env.append_unique('DEFINES', Options.options._defs.split(',')) - #if its already defined in a wscript, don't touch. - if not env['cpp11support']: - env['cpp11support'] = Options.options.enablecpp11 + env['cpp11support'] = True configureCompilerOptions(self) env['PLATFORM'] = sys_platform env['LIB_TYPE'] = Options.options.shared_libs and 'shlib' or 'stlib' + env['declspec_decoration'] = '' + env['windows_dll'] = False + if Options.options.shared_libs and env['COMPILER_CXX'] == 'msvc': + env['declspec_decoration'] = '__declspec(dllexport)' + env['windows_dll'] = True env['install_headers'] = Options.options.install_headers env['install_libs'] = Options.options.install_libs @@ -1439,7 +1438,6 @@ def process_swig_linkage(tsk): tsk.env.LIB = newlib - # # This task generator creates tasks that install an __init__.py # for our python packages. Right now all it does it create an @@ -1711,33 +1709,6 @@ def getSolarisFlags(compilerName): return (bitFlag32, bitFlag64) -def gccHasCpp11(): - try: - output = subprocess.check_output("g++ --help=c++", - stderr=subprocess.STDOUT, - shell=True, - universal_newlines=True) - except subprocess.CalledProcessError: - # If gcc is too old for --help=, then it is too old for C++11 - return False - for line in output.split('\n'): - if re.search(r'-std=c\+\+11', line): - return True - return False - -def iccHasCpp11(): - try: - output = subprocess.check_output("icpc -help", - stderr=subprocess.STDOUT, - shell=True, - universal_newlines=True) - except subprocess.CalledProcessError: - # If icc is too old for -help, then it is too old for C++11 - return False - for line in output.split('\n'): - if re.search(r'c\+\+11', line): - return True - return False def getWscriptTargets(bld, env, path): # Here we're taking a look at the current stack and adding on all the @@ -1802,6 +1773,19 @@ def addSourceTargets(bld, env, path, target): target.targets_to_add += wscriptTargets +def enableWafUnitTests(bld, set_exit_code=True): + """ + If called, run all C++ unit tests after building + :param set_exit_code Flag to set a non-zero exit code if a unit test fails + """ + # TODO: This does not work for Python files. + # The "nice" way to handle this is possibly not + # supported in this version of Waf. + bld.add_post_fun(waf_unit_test.summary) + if set_exit_code: + bld.add_post_fun(waf_unit_test.set_exit_code) + + class SwitchContext(Context.Context): """ Easily switch output directories without reconfiguration. @@ -1876,4 +1860,3 @@ class CPPPackageContext(package, CPPContext): def __init__(self, **kw): self.waf_command = 'python waf' super(CPPPackageContext, self).__init__(**kw) - diff --git a/externals/coda-oss/build/msvs.py b/externals/coda-oss/build/msvs.py index ef059029c..76831a63a 100644 --- a/externals/coda-oss/build/msvs.py +++ b/externals/coda-oss/build/msvs.py @@ -670,7 +670,7 @@ def collect_source(self): for x in include_dirs: if isinstance(x, str): x = tg.path.find_node(x) - if x: + if x and os.path.isdir(x.abspath()): lst = [y for y in x.ant_glob(HEADERS_GLOB, flat=False)] include_files.extend(lst) diff --git a/externals/coda-oss/build/swig.py b/externals/coda-oss/build/swig.py index 8f85a6e9c..2cd253a4c 100644 --- a/externals/coda-oss/build/swig.py +++ b/externals/coda-oss/build/swig.py @@ -195,9 +195,16 @@ def options(opt): opt.add_option('--disable-swig', action='store_false', dest='swig', help='Disable swig') opt.add_option('--swig-version', action='store', dest='swigver', - default=None, help='Specify the minimum swig version') + default='3.0.0', help='Specify the minimum swig version') opt.add_option('--require-swig', action='store_true', dest='require_swig', help='Require swig (configure option)', default=False) + opt.add_option('--python-version-agnostic', action='store_true', + dest='pyver_agnostic', + help='The default behavior for generated Python code is to ' + 'enable Python 3-specific features if the detected Python ' + 'is version 3. Pass this flag to force the generated code ' + 'to work with either version of Python in all cases.') + def configure(conf): if Options.options.swig: @@ -209,3 +216,4 @@ def configure(conf): conf.check_swig_version(minver=swigver) conf.env.SWIGPATH_ST = '-I%s' conf.env.SWIGDEFINES_ST = '-D%s' + conf.env['PYTHON_AGNOSTIC'] = Options.options.pyver_agnostic diff --git a/externals/coda-oss/build/waf b/externals/coda-oss/build/waf index 38e3216dc..81efae1c9 100755 Binary files a/externals/coda-oss/build/waf and b/externals/coda-oss/build/waf differ diff --git a/externals/coda-oss/modules/c++/config/README b/externals/coda-oss/modules/c++/config/README new file mode 100644 index 000000000..05ff588cc --- /dev/null +++ b/externals/coda-oss/modules/c++/config/README @@ -0,0 +1,9 @@ +This module represents the "configure" step for the entire C++ set of libraries. +It exists independently to ensure it can always build first with no +circular dependencies. Its sole responsibility is to generate "coda_oss_config.h" + +Key defines in "coda_oss_config.h": + - CODA_EXPORT: Expands to __declspec(dllexport) when building + shared libraries on Windows + + - __CODA_CPP11: Defined if the library is compiled with C++11 support diff --git a/externals/coda-oss/modules/c++/config/wscript b/externals/coda-oss/modules/c++/config/wscript new file mode 100644 index 000000000..a4160e74b --- /dev/null +++ b/externals/coda-oss/modules/c++/config/wscript @@ -0,0 +1,88 @@ +NAME = 'config' +MAINTAINER = 'jonathan.means@maxar.com' +VERSION = '1.2' +USELIB = 'THREAD DL RT SOCKET' + +import os +from build import writeConfig +from waflib import Utils + +options = distclean = lambda p: None + +def configure(conf): + # this check defines HAVE_CLOCK_GETTIME, undefine it to keep it out of dumpenv + if conf.check_cc(lib='rt', uselib_store='RT', function_name='clock_gettime', header_name='time.h', mandatory=False): + conf.undefine('HAVE_CLOCK_GETTIME') + + # callback function to check for all #defines used by the sys module + def sys_callback(conf): + conf.check_cc(header_name='pthread.h', mandatory=False) + conf.check_cc(header_name='execinfo.h', mandatory=False) + conf.check_cc(function_name='clock_gettime', header_name='time.h', mandatory=False) + conf.check_cc(header_name="atomic.h", mandatory=False) + conf.check_cc(header_name="sys/time.h", mandatory=False) + conf.check_cc(function_name='localtime_r', header_name="time.h", mandatory=False) + conf.check_cc(function_name='gmtime_r', header_name="time.h", mandatory=False) + conf.check_cc(function_name='setenv', header_name="stdlib.h", mandatory=False) + conf.check_cc(function_name='posix_memalign', header_name="stdlib.h", mandatory=False) + conf.check_cc(function_name='memalign', header_name="stdlib.h", mandatory=False) + types_str = ''' + #include + int isBigEndian() + { + long one = 1; + return !(*((char *)(&one))); + } + int main() + { + if (isBigEndian()) printf("bigendian=True\\n"); + else printf("bigendian=False\\n"); + printf("sizeof_size_t=%d\\n", sizeof(size_t)); + return 0; + } + ''' + + # Visual Studio 2013 has nullptr but not constexpr. Need to check for + # both in here to make sure we have full C++11 support... otherwise, + # long-term we may need multiple separate configure checks and + # corresponding defines + cpp11_str = ''' + int main() + { + constexpr void* FOO = nullptr; + } + ''' + #find out the size of some types, etc. + # TODO: This is not using the 32 vs. 64 bit linker flags, so if you're + # building with --enable-32bit on 64 bit Linux, sizeof(size_t) will + # erroneously be 8 here. + output = conf.check(fragment=types_str, execute=1, msg='Checking system type sizes', define_ret=True) + t = Utils.str_to_dict(output or '') + for k, v in t.items(): + try: + v = int(v) + except: + v = v.strip() + if v == 'True': + v = True + elif v == 'False': + v = False + conf.define(k.upper(), v) + conf.check_cxx(fragment=cpp11_str, + execute=1, + msg='Checking for C++11 support', + define_name='__CODA_CPP11', + mandatory=True) + conf.define('CODA_EXPORT', conf.env['declspec_decoration'], quote=False) + + writeConfig(conf, sys_callback, 'coda_oss', + path=os.path.join('include', 'config'), + outfile='coda_oss_config.h') + + +def build(bld): + includeDirname = os.path.join(bld.env['install_includedir'], 'config') + configPathname = bld.path.get_bld().ant_glob('**/coda_oss_config.h') + bld.install_files(dest=includeDirname, + files=configPathname) + bld.module(**globals()) diff --git a/externals/coda-oss/modules/c++/except/include/except/Context.h b/externals/coda-oss/modules/c++/except/include/except/Context.h index 4960fb72d..b9d0e8f5e 100644 --- a/externals/coda-oss/modules/c++/except/include/except/Context.h +++ b/externals/coda-oss/modules/c++/except/include/except/Context.h @@ -1,7 +1,7 @@ /* ========================================================================= - * This file is part of except-c++ + * This file is part of except-c++ * ========================================================================= - * + * * (C) Copyright 2004 - 2014, MDA Information Systems LLC * * except-c++ is free software; you can redistribute it and/or modify @@ -14,8 +14,8 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with this program; If not, + * You should have received a copy of the GNU Lesser General Public + * License along with this program; If not, * see . * */ @@ -26,6 +26,7 @@ #include #include +#include /*! * \file @@ -39,7 +40,7 @@ namespace except * \class Context * \brief The information surrounding an exception or error * - * This class contains information such as the file, line, + * This class contains information such as the file, line, * function and time */ class Context @@ -87,7 +88,7 @@ class Context /*! * Get the function where the exception occurred (may not be available - * \return The function signature + * \return The function signature */ const std::string& getFunction() const { diff --git a/externals/coda-oss/modules/c++/except/wscript b/externals/coda-oss/modules/c++/except/wscript index 5da1951ed..2bc7199e1 100644 --- a/externals/coda-oss/modules/c++/except/wscript +++ b/externals/coda-oss/modules/c++/except/wscript @@ -1,6 +1,7 @@ NAME = 'except' MAINTAINER = 'jmrandol@users.sourceforge.net' VERSION = '1.0' +MODULE_DEPS = 'config' UNITTEST_DEPS = 'sys' options = configure = distclean = lambda p: None diff --git a/externals/coda-oss/modules/c++/math/include/math/Constants.h b/externals/coda-oss/modules/c++/math/include/math/Constants.h index da7b0a9b1..70e9fde30 100644 --- a/externals/coda-oss/modules/c++/math/include/math/Constants.h +++ b/externals/coda-oss/modules/c++/math/include/math/Constants.h @@ -23,9 +23,6 @@ #ifndef __MATH_CONSTANTS_H__ #define __MATH_CONSTANTS_H__ -#include - -#ifdef __CODA_CPP11 #include namespace math { @@ -48,24 +45,4 @@ struct Constants SPEED_OF_LIGHT_METERS_PER_SEC * METERS_TO_FEET; }; } -#else -namespace math -{ -struct Constants -{ - static const double FEET_TO_METERS; - static const double METERS_TO_FEET; - - static const double RADIANS_TO_DEGREES; - static const double DEGREES_TO_RADIANS; - - static const double NAUTICAL_MILES_TO_METERS; - static const double METERS_TO_NAUTICAL_MILES; - static const double NAUTICAL_MILES_TO_FEET; - - static const double SPEED_OF_LIGHT_METERS_PER_SEC; - static const double SPEED_OF_LIGHT_FEET_PER_SEC; -}; -} -#endif #endif diff --git a/externals/coda-oss/modules/c++/math/source/Constants.cpp b/externals/coda-oss/modules/c++/math/source/Constants.cpp deleted file mode 100644 index 6c46c886b..000000000 --- a/externals/coda-oss/modules/c++/math/source/Constants.cpp +++ /dev/null @@ -1,43 +0,0 @@ -/* ========================================================================= - * This file is part of math-c++ - * ========================================================================= - * - * (C) Copyright 2004 - 2014, MDA Information Systems LLC - * - * types-c++ is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation; either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this program; If not, - * see . - * - */ - -#include -#include - -#ifndef __CODA_CPP11 -namespace math -{ -const double Constants::FEET_TO_METERS = 0.3048; -const double Constants::METERS_TO_FEET = 1.0 / Constants::FEET_TO_METERS; -const double Constants::RADIANS_TO_DEGREES = 180.0 / M_PI; -const double Constants::DEGREES_TO_RADIANS = M_PI / 180.0; -const double Constants::NAUTICAL_MILES_TO_METERS = 1852.0; -const double Constants::METERS_TO_NAUTICAL_MILES = - 1.0 / Constants::NAUTICAL_MILES_TO_METERS; -const double Constants::NAUTICAL_MILES_TO_FEET = - Constants::NAUTICAL_MILES_TO_METERS * Constants::METERS_TO_FEET; -const double Constants::SPEED_OF_LIGHT_METERS_PER_SEC = 299792458.0; -const double Constants::SPEED_OF_LIGHT_FEET_PER_SEC = - Constants::SPEED_OF_LIGHT_METERS_PER_SEC * Constants::METERS_TO_FEET; -} - -#endif diff --git a/externals/coda-oss/modules/c++/mem/include/mem/ScopedCopyablePtr.h b/externals/coda-oss/modules/c++/mem/include/mem/ScopedCopyablePtr.h index 92566ee13..f6f50419f 100644 --- a/externals/coda-oss/modules/c++/mem/include/mem/ScopedCopyablePtr.h +++ b/externals/coda-oss/modules/c++/mem/include/mem/ScopedCopyablePtr.h @@ -25,7 +25,7 @@ #include #include -#include +#include namespace mem { @@ -109,10 +109,7 @@ class ScopedCopyablePtr } // explicit operators not supported until C++11 -#ifdef __CODA_CPP11 - explicit -#endif - operator bool() const + explicit operator bool() const { return get() == NULL ? false : true; } diff --git a/externals/coda-oss/modules/c++/mem/include/mem/SharedPtr.h b/externals/coda-oss/modules/c++/mem/include/mem/SharedPtr.h index b0a8a6381..4e54c8909 100644 --- a/externals/coda-oss/modules/c++/mem/include/mem/SharedPtr.h +++ b/externals/coda-oss/modules/c++/mem/include/mem/SharedPtr.h @@ -23,7 +23,7 @@ #ifndef __MEM_SHARED_PTR_H__ #define __MEM_SHARED_PTR_H__ -#include +#include // Adding a forward declaration here in order to have SWIG correctly generate // bindings for SharedPtr. @@ -38,10 +38,6 @@ template class SharedPtr; } -#ifdef __CODA_CPP11 #include -#else -#include -#endif #endif diff --git a/externals/coda-oss/modules/c++/mem/include/mem/SharedPtrLegacy.h b/externals/coda-oss/modules/c++/mem/include/mem/SharedPtrLegacy.h deleted file mode 100644 index 0b5a750dd..000000000 --- a/externals/coda-oss/modules/c++/mem/include/mem/SharedPtrLegacy.h +++ /dev/null @@ -1,184 +0,0 @@ -/* ========================================================================= - * This file is part of mem-c++ - * ========================================================================= - * - * (C) Copyright 2004 - 2014, MDA Information Systems LLC - * - * mem-c++ is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation; either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this program; If not, - * see . - * - */ - -#ifndef __MEM_SHARED_PTR_LEGACY_H__ -#define __MEM_SHARED_PTR_LEGACY_H__ - -#include -#include - -#include - -namespace mem -{ -/*! - * \class SharedPtr - * \brief This class provides RAII for object allocations via new. - * Additionally, it uses thread-safe reference counting so that the - * underlying pointer can be shared among multiple objects. When the - * last SharedPtr goes out of scope, the underlying pointer is - * deleted. - */ -template -class SharedPtr -{ -public: - explicit SharedPtr(T* ptr = NULL) : - mPtr(ptr) - { - // Initially we have a reference count of 1 - // In the constructor, we take ownership of the pointer no matter - // what, so temporarily wrap it in an auto_ptr in case creating the - // atomic counter throws - std::auto_ptr scopedPtr(ptr); - mRefCtr = new sys::AtomicCounter(1); - scopedPtr.release(); - } - - explicit SharedPtr(std::auto_ptr ptr) : - mPtr(ptr.get()) - { - // Initially we have a reference count of 1 - // If this throws, the auto_ptr will clean up the input pointer - // for us - mRefCtr = new sys::AtomicCounter(1); - - // We now own the pointer - ptr.release(); - } - - template - explicit SharedPtr(std::auto_ptr ptr) : - mPtr(ptr.get()) - { - // Initially we have a reference count of 1 - // If this throws, the auto_ptr will clean up the input pointer - // for us - mRefCtr = new sys::AtomicCounter(1); - - // We now own the pointer - ptr.release(); - } - - SharedPtr(const SharedPtr& rhs) : - mRefCtr(rhs.mRefCtr), - mPtr(rhs.mPtr) - { - mRefCtr->increment(); - } - - template - SharedPtr(const SharedPtr& rhs) : - mRefCtr(rhs.mRefCtr), - mPtr(rhs.mPtr) - { - mRefCtr->increment(); - } - - ~SharedPtr() - { - if (mRefCtr->decrementThenGet() == 0) - { - delete mRefCtr; - delete mPtr; - } - } - - const SharedPtr& - operator=(const SharedPtr& rhs) - { - if (this != &rhs) - { - if (mRefCtr->decrementThenGet() == 0) - { - // We were holding the last copy of this data prior to this - // assignment - need to clean it up - delete mRefCtr; - delete mPtr; - } - - mRefCtr = rhs.mRefCtr; - mPtr = rhs.mPtr; - mRefCtr->increment(); - } - - return *this; - } - - T* get() const - { - return mPtr; - } - - T& operator*() const - { - return *mPtr; - } - - T* operator->() const - { - return mPtr; - } - - sys::AtomicCounter::ValueType getCount() const - { - return mRefCtr->get(); - } - - void reset(std::auto_ptr scopedPtr) - { - // NOTE: We need to create newRefCtr on the side before decrementing - // mRefCtr. This way, we can provide the strong exception - // guarantee (i.e. the operation either succeeds or throws - the - // underlying object is always in a good state). - sys::AtomicCounter* const newRefCtr = new sys::AtomicCounter(1); - - if (mRefCtr->decrementThenGet() == 0) - { - // We were holding the last copy of this data prior to this - // reset - need to clean up - delete mRefCtr; - delete mPtr; - } - - mRefCtr = newRefCtr; - mPtr = scopedPtr.release(); - } - - void reset(T* ptr = NULL) - { - // We take ownership of the pointer no matter what, so - // temporarily wrap it in an auto_ptr in case creating - // the atomic counter throws in the underlying method. - reset(std::auto_ptr(ptr)); - } - - // This allows derived classes to be used for construction/assignment - template friend class SharedPtr; - -private: - sys::AtomicCounter* mRefCtr; - T* mPtr; -}; -} - -#endif diff --git a/externals/coda-oss/modules/c++/mem/unittests/test_shared_ptr.cpp b/externals/coda-oss/modules/c++/mem/unittests/test_shared_ptr.cpp index 22749a3d3..6ba8ea66d 100644 --- a/externals/coda-oss/modules/c++/mem/unittests/test_shared_ptr.cpp +++ b/externals/coda-oss/modules/c++/mem/unittests/test_shared_ptr.cpp @@ -20,7 +20,7 @@ * */ -#include +#include #include #include "TestCase.h" @@ -64,12 +64,10 @@ struct BarPtrTest : public FooPtrTest mem::SharedPtr mBarPtr; }; -#ifdef __CODA_CPP11 size_t cpp11Function(std::shared_ptr foo) { return foo->mVal; } -#endif TEST_CASE(testNullCopying) { @@ -258,7 +256,6 @@ TEST_CASE(testCasting) } } -#ifdef __CODA_CPP11 TEST_CASE(testStdSharedPtr) { const mem::SharedPtr fooLegacy(new Foo(123)); @@ -270,8 +267,6 @@ TEST_CASE(testStdSharedPtr) TEST_ASSERT_EQ(cpp11Function(fooLegacy), 123); } -#endif - } int main(int, char**) @@ -283,9 +278,7 @@ int main(int, char**) TEST_CHECK(testAssigning); TEST_CHECK(testSyntax); TEST_CHECK(testCasting); -#ifdef __CODA_CPP11 TEST_CHECK(testStdSharedPtr); -#endif return 0; } diff --git a/externals/coda-oss/modules/c++/mt/include/import/mt.h b/externals/coda-oss/modules/c++/mt/include/import/mt.h index 70452c06e..4e98cb292 100644 --- a/externals/coda-oss/modules/c++/mt/include/import/mt.h +++ b/externals/coda-oss/modules/c++/mt/include/import/mt.h @@ -1,7 +1,7 @@ /* ========================================================================= - * This file is part of mt-c++ + * This file is part of mt-c++ * ========================================================================= - * + * * (C) Copyright 2004 - 2014, MDA Information Systems LLC * * mt-c++ is free software; you can redistribute it and/or modify @@ -14,8 +14,8 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with this program; If not, + * You should have received a copy of the GNU Lesser General Public + * License along with this program; If not, * see . * */ @@ -43,7 +43,5 @@ #include "mt/CPUAffinityInitializer.h" #include "mt/CPUAffinityThreadInitializer.h" -#include "mt/LinuxCPUAffinityInitializer.h" -#include "mt/LinuxCPUAffinityThreadInitializer.h" #endif diff --git a/externals/coda-oss/modules/c++/mt/include/mt/AbstractCPUAffinityInitializer.h b/externals/coda-oss/modules/c++/mt/include/mt/AbstractCPUAffinityInitializer.h new file mode 100644 index 000000000..847ee9784 --- /dev/null +++ b/externals/coda-oss/modules/c++/mt/include/mt/AbstractCPUAffinityInitializer.h @@ -0,0 +1,63 @@ +/* ========================================================================= + * This file is part of mt-c++ + * ========================================================================= + * + * (C) Copyright 2004 - 2019, MDA Information Systems LLC + * + * mt-c++ is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this program; If not, + * see . + * + */ + + +#ifndef __MT_ABSTRACT_CPU_AFFINITY_INITIALIZER_H__ +#define __MT_ABSTRACT_CPU_AFFINITY_INITIALIZER_H__ + +#include + +namespace mt +{ +/*! + * \class AbstractCPUAffinityInitializer + * \brief Abstract class for providing thread-level affinity initializers. + * Should be used by the primary thread to generate data for + * child threads. + */ +class AbstractCPUAffinityInitializer +{ +public: + virtual ~AbstractCPUAffinityInitializer() {} + + /*! + * \returns a new thread initializer. In general, this should return + * a different affinity initializer each time it is called. + */ + std::auto_ptr newThreadInitializer() + { + return std::auto_ptr( + newThreadInitializerImpl()); + } + +private: + // To allow for covariant auto_ptrs, this private function can be + // implemented in derived classes to return a raw, unmanaged pointer + // with the override having a covariant return type. + // Using name hiding, we can define newThreadInitializer() implementations + // that wrap newThreadInitializerImpl() in the appropriate derived + // class return type. This should be done in all derived classes. + virtual AbstractCPUAffinityThreadInitializer* newThreadInitializerImpl() = 0; +}; +} + +#endif diff --git a/externals/coda-oss/modules/c++/mt/source/LinuxCPUAffinityInitializer.cpp b/externals/coda-oss/modules/c++/mt/include/mt/AbstractCPUAffinityThreadInitializer.h similarity index 58% rename from externals/coda-oss/modules/c++/mt/source/LinuxCPUAffinityInitializer.cpp rename to externals/coda-oss/modules/c++/mt/include/mt/AbstractCPUAffinityThreadInitializer.h index a751d3023..781746786 100644 --- a/externals/coda-oss/modules/c++/mt/source/LinuxCPUAffinityInitializer.cpp +++ b/externals/coda-oss/modules/c++/mt/include/mt/AbstractCPUAffinityThreadInitializer.h @@ -1,7 +1,7 @@ /* ========================================================================= - * This file is part of mt-c++ + * This file is part of mt-c++ * ========================================================================= - * + * * (C) Copyright 2004 - 2014, MDA Information Systems LLC * * mt-c++ is free software; you can redistribute it and/or modify @@ -14,26 +14,32 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with this program; If not, + * You should have received a copy of the GNU Lesser General Public + * License along with this program; If not, * see . * */ -#include "mt/LinuxCPUAffinityInitializer.h" - -#if !defined(__APPLE_CC__) -#if defined(__linux) || defined(__linux__) +#ifndef __MT_ABSTRACT_CPU_AFFINITY_THREAD_INITIALIZER_H__ +#define __MT_ABSTRACT_CPU_AFFINITY_THREAD_INITIALIZER_H__ -cpu_set_t mt::LinuxCPUAffinityInitializer::nextCPU() +namespace mt { - cpu_set_t affinityMask; - CPU_ZERO(&affinityMask); - CPU_SET(mNextCPU, &affinityMask); - ++mNextCPU; - return affinityMask; -} +/*! + * \class AbstractCPUAffinityThreadInitializer + * \brief Abstract class for setting the CPU affinity of a thread + */ +class AbstractCPUAffinityThreadInitializer +{ +public: + AbstractCPUAffinityThreadInitializer() {} + virtual ~AbstractCPUAffinityThreadInitializer() {} -#endif + /*! + * Set the CPU affinity of the currently running thread + */ + virtual void initialize() = 0; +}; +} #endif diff --git a/externals/coda-oss/modules/c++/mt/include/mt/AbstractTiedThreadPool.h b/externals/coda-oss/modules/c++/mt/include/mt/AbstractTiedThreadPool.h index f3ce72b44..3099f4262 100644 --- a/externals/coda-oss/modules/c++/mt/include/mt/AbstractTiedThreadPool.h +++ b/externals/coda-oss/modules/c++/mt/include/mt/AbstractTiedThreadPool.h @@ -1,8 +1,8 @@ /* ========================================================================= - * This file is part of mt-c++ + * This file is part of mt-c++ * ========================================================================= - * - * (C) Copyright 2004 - 2014, MDA Information Systems LLC + * + * (C) Copyright 2004 - 2019, MDA Information Systems LLC * * mt-c++ is free software; you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License as published by @@ -14,8 +14,8 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with this program; If not, + * You should have received a copy of the GNU Lesser General Public + * License along with this program; If not, * see . * */ @@ -35,39 +35,45 @@ class AbstractTiedThreadPool : public AbstractThreadPool { public: - AbstractTiedThreadPool(unsigned short numThreads = 0) : AbstractThreadPool(numThreads) {} + AbstractTiedThreadPool(unsigned short numThreads = 0) : + AbstractThreadPool(numThreads) + { + } + + virtual ~AbstractTiedThreadPool(){} + + virtual void initialize(CPUAffinityInitializer* affinityInit = NULL) + { + mAffinityInit = affinityInit; + } - virtual ~AbstractTiedThreadPool(){} + virtual std::auto_ptr + getCPUAffinityThreadInitializer() + { + std::auto_ptr threadInit(NULL); - virtual void initialize(CPUAffinityInitializer* affinityInit = NULL) - { - mAffinityInit = affinityInit; - } - virtual CPUAffinityThreadInitializer* getCPUAffinityThreadInitializer() - { - CPUAffinityThreadInitializer* threadInit = NULL; - - // If we were passed a schematic - // for initializing thread affinity... - if (mAffinityInit) - { - threadInit = mAffinityInit->newThreadInitializer(); - } - return threadInit; - } + // If we were passed a schematic + // for initializing thread affinity... + if (mAffinityInit) + { + threadInit = mAffinityInit->newThreadInitializer(); + } + return threadInit; + } - virtual mt::WorkerThread* newWorker() - { - return newTiedWorker(&this->mRequestQueue, - getCPUAffinityThreadInitializer()); - } + virtual mt::WorkerThread* newWorker() + { + return newTiedWorker(&this->mRequestQueue, + getCPUAffinityThreadInitializer()); + } protected: - virtual mt::TiedWorkerThread* newTiedWorker(mt::RequestQueue* q, - CPUAffinityThreadInitializer* init) = 0; + virtual mt::TiedWorkerThread* + newTiedWorker(mt::RequestQueue* q, + std::auto_ptr init) = 0; private: - CPUAffinityInitializer* mAffinityInit; + CPUAffinityInitializer* mAffinityInit; }; } diff --git a/externals/coda-oss/modules/c++/mt/include/mt/CPUAffinityInitializer.h b/externals/coda-oss/modules/c++/mt/include/mt/CPUAffinityInitializer.h index b93843457..6fe868bea 100644 --- a/externals/coda-oss/modules/c++/mt/include/mt/CPUAffinityInitializer.h +++ b/externals/coda-oss/modules/c++/mt/include/mt/CPUAffinityInitializer.h @@ -1,8 +1,8 @@ /* ========================================================================= - * This file is part of mt-c++ + * This file is part of mt-c++ * ========================================================================= - * - * (C) Copyright 2004 - 2014, MDA Information Systems LLC + * + * (C) Copyright 2004 - 2019, MDA Information Systems LLC * * mt-c++ is free software; you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License as published by @@ -14,8 +14,8 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with this program; If not, + * You should have received a copy of the GNU Lesser General Public + * License along with this program; If not, * see . * */ @@ -24,17 +24,24 @@ #ifndef __MT_CPU_AFFINITY_INITIALIZER_H__ #define __MT_CPU_AFFINITY_INITIALIZER_H__ -#include "mt/CPUAffinityThreadInitializer.h" +#include + +#if defined(WIN32) +#include +namespace mt +{ +typedef CPUAffinityInitializerWin32 CPUAffinityInitializer; +} +#endif +#if !defined(__APPLE_CC__) +#if defined(__linux) || defined(__linux__) +#include namespace mt { - class CPUAffinityInitializer - { - public: - CPUAffinityInitializer() {} - virtual ~CPUAffinityInitializer() {} - virtual CPUAffinityThreadInitializer* newThreadInitializer() = 0; - }; +typedef CPUAffinityInitializerLinux CPUAffinityInitializer; } +#endif +#endif #endif diff --git a/externals/coda-oss/modules/c++/mt/include/mt/CPUAffinityInitializerLinux.h b/externals/coda-oss/modules/c++/mt/include/mt/CPUAffinityInitializerLinux.h new file mode 100644 index 000000000..20dca0200 --- /dev/null +++ b/externals/coda-oss/modules/c++/mt/include/mt/CPUAffinityInitializerLinux.h @@ -0,0 +1,91 @@ +/* ========================================================================= + * This file is part of mt-c++ + * ========================================================================= + * + * (C) Copyright 2004 - 2019, MDA Information Systems LLC + * + * mt-c++ is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this program; If not, + * see . + * + */ + + +#ifndef __MT_CPU_AFFINITY_INITIALIZER_LINUX_H__ +#define __MT_CPU_AFFINITY_INITIALIZER_LINUX_H__ + +#if !defined(__APPLE_CC__) +#if defined(__linux) || defined(__linux__) + +#include +#include + +#include +#include +#include + +namespace mt +{ +struct AbstractNextCPUProviderLinux +{ + virtual std::auto_ptr nextCPU() = 0; +}; + +/*! + * \class CPUAffinityInitializerLinux + * \brief Linux-specific class for providing thread-level affinity initializers. + */ +class CPUAffinityInitializerLinux : public AbstractCPUAffinityInitializer +{ +public: + + /*! + * Constructor that uses the available CPUs (possibly restricted + * via taskset) to set affinities + */ + CPUAffinityInitializerLinux(); + + /*! + * Constructor that uses all online CPUs to incrementally update + * to the next CPU, starting from 'initialOffset'. + * + * \param initialOffset Initial CPU to start using when pinning threads + */ + CPUAffinityInitializerLinux(int initialOffset); + + /*! + * \throws if there are no more available CPUs to bind to + * \returns a new CPUAffinityInitializerLinux for the next available + * CPU that can be bound to. + */ + std::auto_ptr newThreadInitializer() + { + return std::auto_ptr( + newThreadInitializerImpl()); + } + +private: + + virtual CPUAffinityThreadInitializerLinux* newThreadInitializerImpl() + { + return new CPUAffinityThreadInitializerLinux(mCPUProvider->nextCPU()); + } + + std::auto_ptr mCPUProvider; +}; +} + +#endif +#endif +#endif + diff --git a/externals/coda-oss/modules/c++/mt/include/mt/CPUAffinityInitializerWin32.h b/externals/coda-oss/modules/c++/mt/include/mt/CPUAffinityInitializerWin32.h new file mode 100644 index 000000000..5fdf118eb --- /dev/null +++ b/externals/coda-oss/modules/c++/mt/include/mt/CPUAffinityInitializerWin32.h @@ -0,0 +1,62 @@ +/* ========================================================================= + * This file is part of mt-c++ + * ========================================================================= + * + * (C) Copyright 2004 - 2019, MDA Information Systems LLC + * + * mt-c++ is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this program; If not, + * see . + * + */ + + +#ifndef __MT_CPU_AFFINITY_INITIALIZER_WIN32_H__ +#define __MT_CPU_AFFINITY_INITIALIZER_WIN32_H__ + +#if defined(WIN32) + +#include +#include + +namespace mt +{ +/*! + * \class CPUAffinityInitializerWin32 + * \brief Windows-specific class for providing thread-level affinity initializers. + * \todo This is a stub implementation that doesn't do anything. Make this work. + */ +class CPUAffinityInitializerWin32 : public AbstractCPUAffinityInitializer +{ +public: + /*! + * \todo Not yet implemented + * \returns NULL + */ + std::auto_ptr newThreadInitializer() + { + return std::auto_ptr( + newThreadInitializerImpl()); + } + +private: + virtual CPUAffinityThreadInitializerWin32* newThreadInitializerImpl() + { + return NULL; + } +}; +} + +#endif +#endif + diff --git a/externals/coda-oss/modules/c++/mt/include/mt/CPUAffinityThreadInitializer.h b/externals/coda-oss/modules/c++/mt/include/mt/CPUAffinityThreadInitializer.h index 1663051b0..17fbbb980 100644 --- a/externals/coda-oss/modules/c++/mt/include/mt/CPUAffinityThreadInitializer.h +++ b/externals/coda-oss/modules/c++/mt/include/mt/CPUAffinityThreadInitializer.h @@ -1,8 +1,8 @@ /* ========================================================================= - * This file is part of mt-c++ + * This file is part of mt-c++ * ========================================================================= - * - * (C) Copyright 2004 - 2014, MDA Information Systems LLC + * + * (C) Copyright 2004 - 2019, MDA Information Systems LLC * * mt-c++ is free software; you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License as published by @@ -14,8 +14,8 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with this program; If not, + * You should have received a copy of the GNU Lesser General Public + * License along with this program; If not, * see . * */ @@ -24,14 +24,24 @@ #ifndef __MT_CPU_AFFINITY_THREAD_INITIALIZER_H__ #define __MT_CPU_AFFINITY_THREAD_INITIALIZER_H__ +#include + +#if defined(WIN32) +#include +namespace mt +{ +typedef CPUAffinityThreadInitializerWin32 CPUAffinityThreadInitializer; +} +#endif + +#if !defined(__APPLE_CC__) +#if defined(__linux) || defined(__linux__) +#include namespace mt { - class CPUAffinityThreadInitializer - { - public: - CPUAffinityThreadInitializer() {} - virtual ~CPUAffinityThreadInitializer() {} - virtual void initialize() = 0; - }; +typedef CPUAffinityThreadInitializerLinux CPUAffinityThreadInitializer; } #endif +#endif + +#endif diff --git a/externals/coda-oss/modules/c++/mt/include/mt/CPUAffinityThreadInitializerLinux.h b/externals/coda-oss/modules/c++/mt/include/mt/CPUAffinityThreadInitializerLinux.h new file mode 100644 index 000000000..b1593f063 --- /dev/null +++ b/externals/coda-oss/modules/c++/mt/include/mt/CPUAffinityThreadInitializerLinux.h @@ -0,0 +1,70 @@ +/* ========================================================================= + * This file is part of mt-c++ + * ========================================================================= + * + * (C) Copyright 2004 - 2019, MDA Information Systems LLC + * + * mt-c++ is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this program; If not, + * see . + * + */ + + +#ifndef __MT_CPU_AFFINITY_THREAD_INITIALIZER_LINUX_H__ +#define __MT_CPU_AFFINITY_THREAD_INITIALIZER_LINUX_H__ + +#if !defined(__APPLE_CC__) +#if defined(__linux) || defined(__linux__) + +#include + +#include +#include + +namespace mt +{ +/*! + * \class CPUAffinityThreadInitializerLinux + * \brief Linux-specific setting of the CPU affinity of a thread + */ +class CPUAffinityThreadInitializerLinux : + public AbstractCPUAffinityThreadInitializer +{ +public: + + /*! + * Constructor + * + * \param cpu A ScopedCPUMaskUnix object corresponding to the + * affinity mask for the CPUs that this thread + * is allowed to bind to + */ + CPUAffinityThreadInitializerLinux( + std::auto_ptr cpu); + + /*! + * Attempt to bind to the affinity mask given during construction + * + * \throws if setting the thread affinity fails + */ + virtual void initialize(); + +private: + std::auto_ptr mCPU; +}; +} + +#endif +#endif +#endif diff --git a/externals/coda-oss/modules/c++/mt/include/mt/LinuxCPUAffinityInitializer.h b/externals/coda-oss/modules/c++/mt/include/mt/CPUAffinityThreadInitializerWin32.h similarity index 50% rename from externals/coda-oss/modules/c++/mt/include/mt/LinuxCPUAffinityInitializer.h rename to externals/coda-oss/modules/c++/mt/include/mt/CPUAffinityThreadInitializerWin32.h index 2184b8e7c..0bbd85271 100644 --- a/externals/coda-oss/modules/c++/mt/include/mt/LinuxCPUAffinityInitializer.h +++ b/externals/coda-oss/modules/c++/mt/include/mt/CPUAffinityThreadInitializerWin32.h @@ -1,8 +1,8 @@ /* ========================================================================= - * This file is part of mt-c++ + * This file is part of mt-c++ * ========================================================================= - * - * (C) Copyright 2004 - 2014, MDA Information Systems LLC + * + * (C) Copyright 2004 - 2019, MDA Information Systems LLC * * mt-c++ is free software; you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License as published by @@ -14,44 +14,33 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with this program; If not, + * You should have received a copy of the GNU Lesser General Public + * License along with this program; If not, * see . * */ -#ifndef __MT_LINUX_CPU_AFFINITY_INITIALIZER_H__ -#define __MT_LINUX_CPU_AFFINITY_INITIALIZER_H__ - -#if !defined(__APPLE_CC__) -#if defined(__linux) || defined(__linux__) +#ifndef __MT_CPU_AFFINITY_THREAD_INITIALIZER_WIN32_H__ +#define __MT_CPU_AFFINITY_THREAD_INITIALIZER_WIN32_H__ +#if defined(WIN32) -#include "mt/CPUAffinityInitializer.h" -#include "mt/LinuxCPUAffinityThreadInitializer.h" +#include namespace mt { - class LinuxCPUAffinityInitializer : public mt::CPUAffinityInitializer - { - int mNextCPU; - cpu_set_t nextCPU(); - public: - LinuxCPUAffinityInitializer(int initialOffset) : - mNextCPU(initialOffset) {} - - ~LinuxCPUAffinityInitializer() {} - - LinuxCPUAffinityThreadInitializer* newThreadInitializer() - { - return new LinuxCPUAffinityThreadInitializer(nextCPU()); - } - - }; +/*! + * \class CPUAffinityThreadInitializerWin32 + * \brief Windows-specific setting of the CPU affinity of a thread + * \todo This is a stub implementation that doesn't do anything. Make this work. + */ +class CPUAffinityThreadInitializerWin32 : public AbstractCPUAffinityThreadInitializer +{ +public: + virtual void initialize() {} +}; } #endif #endif -#endif - diff --git a/externals/coda-oss/modules/c++/mt/include/mt/GenerationThreadPool.h b/externals/coda-oss/modules/c++/mt/include/mt/GenerationThreadPool.h index 832cb8ad1..0ccd83896 100644 --- a/externals/coda-oss/modules/c++/mt/include/mt/GenerationThreadPool.h +++ b/externals/coda-oss/modules/c++/mt/include/mt/GenerationThreadPool.h @@ -80,9 +80,10 @@ namespace mt handler->setSemaphore(&mGenerationSync); if (mAffinityInit) - handler->setAffinityInit(mAffinityInit->newThreadInitializer()); - - + { + handler->setAffinityInit(mAffinityInit->newThreadInitializer().release()); + } + return handler; } diff --git a/externals/coda-oss/modules/c++/mt/include/mt/ThreadGroup.h b/externals/coda-oss/modules/c++/mt/include/mt/ThreadGroup.h index 0c5f87130..5187338c9 100644 --- a/externals/coda-oss/modules/c++/mt/include/mt/ThreadGroup.h +++ b/externals/coda-oss/modules/c++/mt/include/mt/ThreadGroup.h @@ -1,7 +1,7 @@ /* ========================================================================= - * This file is part of mt-c++ + * This file is part of mt-c++ * ========================================================================= - * + * * (C) Copyright 2004 - 2014, MDA Information Systems LLC * * mt-c++ is free software; you can redistribute it and/or modify @@ -14,8 +14,8 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with this program; If not, + * You should have received a copy of the GNU Lesser General Public + * License along with this program; If not, * see . * */ @@ -26,13 +26,16 @@ #include #include #include -#include "sys/Runnable.h" -#include "sys/Thread.h" -#include "mem/SharedPtr.h" -#include "except/Error.h" -#include +#include +#include +#include +#include +#include +#include "mt/mt_config.h" +#include +#include namespace mt { @@ -50,42 +53,85 @@ class ThreadGroup { public: - //! Constructor. - ThreadGroup(); - + /*! + * Constructor. + * \param pinToCPU Optional flag specifying whether CPU pinning + * should occur, using a CPUAffinityInitializer. + * If MT_DEFAULT_PINNING is defined, defaults to true + * (enable affinity-based thread pinning). + * Otherwise, defaults to false (no pinning). + */ + ThreadGroup(bool pinToCPU = getDefaultPinToCPU()); + /*! * Destructor. Attempts to join all threads. */ ~ThreadGroup(); - + /*! * Creates and starts a thread from a sys::Runnable. * \param runnable pointer to sys::Runnable */ void createThread(sys::Runnable *runnable); - + /*! * Creates and starts a thread from a sys::Runnable. * \param runnable auto_ptr to sys::Runnable */ void createThread(std::auto_ptr runnable); - + /*! * Waits for all threads to complete. */ void joinAll(); + /*! + * Checks whether CPU pinning is set (i.e. whether mAffinityInit is NULL) + * + * \return true if CPU pinning is set, false otherwise + */ + bool isPinToCPUEnabled() const; + + /*! + * Gets the current default value for CPU pinning + * + * \return true if CPU pinning is enabled by default, false otherwise + */ + static bool getDefaultPinToCPU(); + + /*! + * Sets the default value for CPU pinning + * + * \param newDefault the new default value for CPU pinning + */ + static void setDefaultPinToCPU(bool newDefault); + private: - std::vector > mThreads; + std::auto_ptr mAffinityInit; size_t mLastJoined; + std::vector > mThreads; std::vector mExceptions; sys::Mutex mMutex; + /*! + * The default setting for pinToCPU, used in the ThreadGroup constructor + * Can't set non-const static values in the header, so set at the top of + * the implementation + */ + static bool DEFAULT_PIN_TO_CPU; + /*! * Adds an exception to the mExceptions vector */ void addException(const except::Exception& ex); + /*! + * \returns the next available thread initializer provided by + * the internal CPUAffinityInitializer. If no initializer + * was created, will return NULL. + */ + std::auto_ptr getNextInitializer(); + /*! * \class ThreadGroupRunnable * @@ -96,9 +142,22 @@ class ThreadGroup { public: - //! Constructor. - ThreadGroupRunnable(std::auto_ptr runnable, - mt::ThreadGroup& parentThreadGroup); + /*! + * Constructor. + * \param runnable sys::Runnable object that will be executed by + * the current thread + * \param parentThreadGroup ThreadGroup object that spawned + * this ThreadGroupRunnable + * \param threadInit Affinity-based initializer for the thread + * that controls which CPUs the runnable is allowed + * to execute on. If NULL, no affinity preferences + * will be enforced. + */ + ThreadGroupRunnable( + std::auto_ptr runnable, + mt::ThreadGroup& parentThreadGroup, + std::auto_ptr threadInit = + std::auto_ptr(NULL)); /*! * Call run() on the Runnable passed to createThread @@ -108,7 +167,7 @@ class ThreadGroup private: std::auto_ptr mRunnable; mt::ThreadGroup& mParentThreadGroup; - + std::auto_ptr mCPUInit; }; }; diff --git a/externals/coda-oss/modules/c++/mt/include/mt/TiedWorkerThread.h b/externals/coda-oss/modules/c++/mt/include/mt/TiedWorkerThread.h index aa3d1f1d3..397b335ef 100644 --- a/externals/coda-oss/modules/c++/mt/include/mt/TiedWorkerThread.h +++ b/externals/coda-oss/modules/c++/mt/include/mt/TiedWorkerThread.h @@ -1,8 +1,8 @@ /* ========================================================================= - * This file is part of mt-c++ + * This file is part of mt-c++ * ========================================================================= - * - * (C) Copyright 2004 - 2014, MDA Information Systems LLC + * + * (C) Copyright 2004 - 2019, MDA Information Systems LLC * * mt-c++ is free software; you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License as published by @@ -14,8 +14,8 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with this program; If not, + * You should have received a copy of the GNU Lesser General Public + * License along with this program; If not, * see . * */ @@ -37,33 +37,28 @@ template class TiedWorkerThread : public mt::WorkerThread { public: - TiedWorkerThread(mt::RequestQueue* requestQueue, - CPUAffinityThreadInitializer* cpuAffinityInit = NULL): - mt::WorkerThread(requestQueue), - mCPUAffinityInit(cpuAffinityInit){} + TiedWorkerThread( + mt::RequestQueue* requestQueue, + std::auto_ptr cpuAffinityInit = + std::auto_ptr(NULL)) : + mt::WorkerThread(requestQueue), + mCPUAffinityInit(cpuAffinityInit) + { + } - virtual ~TiedWorkerThread() - { - if (mCPUAffinityInit) - { - delete mCPUAffinityInit; - mCPUAffinityInit = NULL; - } - } - - virtual void initialize() - { - if (mCPUAffinityInit) - { - mCPUAffinityInit->initialize(); - } - } + virtual void initialize() + { + if (mCPUAffinityInit.get()) + { + mCPUAffinityInit->initialize(); + } + } - virtual void performTask(Request_T& request) = 0; + virtual void performTask(Request_T& request) = 0; private: - TiedWorkerThread(); - CPUAffinityThreadInitializer* mCPUAffinityInit; + TiedWorkerThread(); + std::auto_ptr mCPUAffinityInit; }; } diff --git a/externals/coda-oss/modules/c++/mt/source/CPUAffinityInitializerLinux.cpp b/externals/coda-oss/modules/c++/mt/source/CPUAffinityInitializerLinux.cpp new file mode 100644 index 000000000..df005cf52 --- /dev/null +++ b/externals/coda-oss/modules/c++/mt/source/CPUAffinityInitializerLinux.cpp @@ -0,0 +1,113 @@ +/* ========================================================================= + * This file is part of mt-c++ + * ========================================================================= + * + * (C) Copyright 2004 - 2019, MDA Information Systems LLC + * + * mt-c++ is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this program; If not, + * see . + * + */ + + +#if !defined(__APPLE_CC__) +#if defined(__linux) || defined(__linux__) + +#include + +#include +#include +#include +#include + +namespace +{ +std::vector mergeAvailableCPUs() +{ + std::vector physicalCPUs; + std::vector htCPUs; + sys::OS().getAvailableCPUs(physicalCPUs, htCPUs); + + // Merge physical CPUs first so that + // traversal visits them before hyperthreaded CPUs + std::vector mergedCPUs; + mergedCPUs.reserve(physicalCPUs.size() + htCPUs.size()); + mergedCPUs.insert(mergedCPUs.end(), physicalCPUs.begin(), physicalCPUs.end()); + mergedCPUs.insert(mergedCPUs.end(), htCPUs.begin(), htCPUs.end()); + return mergedCPUs; +} +} + +namespace mt +{ +class AvailableCPUProvider : public AbstractNextCPUProviderLinux +{ +public: + AvailableCPUProvider() : + mCPUs(mergeAvailableCPUs()), + mNextCPUIndex(0) + { + } + + virtual std::auto_ptr nextCPU() + { + if (mNextCPUIndex >= mCPUs.size()) + { + std::ostringstream msg; + msg << "No more CPUs available (size = " << mCPUs.size() << ")"; + throw except::Exception(Ctxt(msg.str())); + } + + std::auto_ptr mask(new sys::ScopedCPUMaskUnix()); + CPU_SET_S(mCPUs.at(mNextCPUIndex++), mask->getSize(), mask->getMask()); + return std::auto_ptr(mask); + } + +private: + const std::vector mCPUs; + size_t mNextCPUIndex; +}; + +class OffsetCPUProvider : public AbstractNextCPUProviderLinux +{ +public: + OffsetCPUProvider(int initialOffset) : + mNextCPU(initialOffset) + { + } + + virtual std::auto_ptr nextCPU() + { + std::auto_ptr mask(new sys::ScopedCPUMaskUnix()); + CPU_SET_S(mNextCPU++, mask->getSize(), mask->getMask()); + return std::auto_ptr(mask); + } + +private: + int mNextCPU; +}; + +CPUAffinityInitializerLinux::CPUAffinityInitializerLinux() : + mCPUProvider(new AvailableCPUProvider()) +{ +} + +CPUAffinityInitializerLinux::CPUAffinityInitializerLinux(int initialOffset) : + mCPUProvider(new OffsetCPUProvider(initialOffset)) +{ +} +} + +#endif +#endif diff --git a/externals/coda-oss/modules/c++/mt/include/mt/LinuxCPUAffinityThreadInitializer.h b/externals/coda-oss/modules/c++/mt/source/CPUAffinityThreadInitializerLinux.cpp similarity index 59% rename from externals/coda-oss/modules/c++/mt/include/mt/LinuxCPUAffinityThreadInitializer.h rename to externals/coda-oss/modules/c++/mt/source/CPUAffinityThreadInitializerLinux.cpp index a8456f0f8..1e36f6ff2 100644 --- a/externals/coda-oss/modules/c++/mt/include/mt/LinuxCPUAffinityThreadInitializer.h +++ b/externals/coda-oss/modules/c++/mt/source/CPUAffinityThreadInitializerLinux.cpp @@ -1,8 +1,8 @@ /* ========================================================================= - * This file is part of mt-c++ + * This file is part of mt-c++ * ========================================================================= - * - * (C) Copyright 2004 - 2014, MDA Information Systems LLC + * + * (C) Copyright 2004 - 2019, MDA Information Systems LLC * * mt-c++ is free software; you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License as published by @@ -14,40 +14,41 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with this program; If not, + * You should have received a copy of the GNU Lesser General Public + * License along with this program; If not, * see . * */ -#ifndef __MT_LINUX_CPU_AFFINITY_THREAD_INITIALIZER_H__ -#define __MT_LINUX_CPU_AFFINITY_THREAD_INITIALIZER_H__ - #if !defined(__APPLE_CC__) #if defined(__linux) || defined(__linux__) #include #include -#include #include -#define gettid() syscall(SYS_gettid) -#include -#include "mt/CPUAffinityThreadInitializer.h" +#include +#include +#include namespace mt { - class LinuxCPUAffinityThreadInitializer : public mt::CPUAffinityThreadInitializer - { - cpu_set_t mCPU; - public: - LinuxCPUAffinityThreadInitializer(const cpu_set_t& cpu); - void initialize(); - }; +CPUAffinityThreadInitializerLinux::CPUAffinityThreadInitializerLinux( + std::auto_ptr cpu) : + mCPU(cpu) +{ +} + +void CPUAffinityThreadInitializerLinux::initialize() +{ + pid_t tid = syscall(SYS_gettid); + if (::sched_setaffinity(tid, mCPU->getSize(), mCPU->getMask()) == -1) + { + throw except::Exception(Ctxt("Failed setting processor affinity")); + } +} } - -#endif #endif #endif diff --git a/externals/coda-oss/modules/c++/mt/source/LinuxCPUAffinityThreadInitializer.cpp b/externals/coda-oss/modules/c++/mt/source/LinuxCPUAffinityThreadInitializer.cpp deleted file mode 100644 index 4556d1f31..000000000 --- a/externals/coda-oss/modules/c++/mt/source/LinuxCPUAffinityThreadInitializer.cpp +++ /dev/null @@ -1,54 +0,0 @@ -/* ========================================================================= - * This file is part of mt-c++ - * ========================================================================= - * - * (C) Copyright 2004 - 2014, MDA Information Systems LLC - * - * mt-c++ is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation; either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this program; If not, - * see . - * - */ - - -#include "mt/LinuxCPUAffinityThreadInitializer.h" - -#if !defined(__APPLE_CC__) -#if defined(__linux) || defined(__linux__) - -mt::LinuxCPUAffinityThreadInitializer:: -LinuxCPUAffinityThreadInitializer(const cpu_set_t& cpu) -{ - for (int i = 0; i < CPU_SETSIZE; ++i) - { - CPU_CLR(i, &mCPU); - if (CPU_ISSET(i, &cpu)) - CPU_SET(i, &mCPU); - } - -} - -void mt::LinuxCPUAffinityThreadInitializer::initialize() -{ - pid_t tid = 0; - tid = ::gettid(); - if ( ::sched_setaffinity(tid, sizeof(mCPU), &mCPU) == -1 ) - { - sys::Err e; - std::ostringstream errStr; - errStr << "Failed setting processor affinity: " << e.toString(); - throw except::Exception(Ctxt(errStr.str())); - } -} -#endif -#endif diff --git a/externals/coda-oss/modules/c++/mt/source/ThreadGroup.cpp b/externals/coda-oss/modules/c++/mt/source/ThreadGroup.cpp index 975c39fce..8893b0ce8 100644 --- a/externals/coda-oss/modules/c++/mt/source/ThreadGroup.cpp +++ b/externals/coda-oss/modules/c++/mt/source/ThreadGroup.cpp @@ -1,7 +1,7 @@ /* ========================================================================= - * This file is part of mt-c++ + * This file is part of mt-c++ * ========================================================================= - * + * * (C) Copyright 2004 - 2014, MDA Information Systems LLC * * mt-c++ is free software; you can redistribute it and/or modify @@ -14,21 +14,32 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with this program; If not, + * You should have received a copy of the GNU Lesser General Public + * License along with this program; If not, * see . * */ + #include #include -mt::ThreadGroup::ThreadGroup() : +namespace mt +{ +#if defined(MT_DEFAULT_PINNING) + bool ThreadGroup::DEFAULT_PIN_TO_CPU = true; +#else + bool ThreadGroup::DEFAULT_PIN_TO_CPU = false; +#endif + + +ThreadGroup::ThreadGroup(bool pinToCPU) : + mAffinityInit(pinToCPU ? new CPUAffinityInitializer() : NULL), mLastJoined(0) { } -mt::ThreadGroup::~ThreadGroup() +ThreadGroup::~ThreadGroup() { try { @@ -40,21 +51,28 @@ mt::ThreadGroup::~ThreadGroup() } } -void mt::ThreadGroup::createThread(sys::Runnable *runnable) +void ThreadGroup::createThread(sys::Runnable *runnable) { createThread(std::auto_ptr(runnable)); } -void mt::ThreadGroup::createThread(std::auto_ptr runnable) +void ThreadGroup::createThread(std::auto_ptr runnable) { - std::auto_ptr internalRunnable(new ThreadGroupRunnable(runnable, *this)); + // Note: If getNextInitializer throws, any previously created + // threads may never finish if cross-thread communication is used. + std::auto_ptr internalRunnable( + new ThreadGroupRunnable( + runnable, + *this, + getNextInitializer())); + mem::SharedPtr thread(new sys::Thread(internalRunnable.get())); internalRunnable.release(); mThreads.push_back(thread); thread->start(); } -void mt::ThreadGroup::joinAll() +void ThreadGroup::joinAll() { bool failed = false; // Keep track of which threads we've already joined. @@ -69,7 +87,7 @@ void mt::ThreadGroup::joinAll() failed = true; } } - + if (!mExceptions.empty()) { std::string messageString("Exceptions thrown from ThreadGroup in the following order:\n"); @@ -84,11 +102,11 @@ void mt::ThreadGroup::joinAll() throw except::Error(Ctxt("ThreadGroup could not be joined")); } -void mt::ThreadGroup::addException(const except::Exception& ex) +void ThreadGroup::addException(const except::Exception& ex) { try { - mt::CriticalSection pushLock(&mMutex); + CriticalSection pushLock(&mMutex); mExceptions.push_back(ex); } catch(...) @@ -97,16 +115,35 @@ void mt::ThreadGroup::addException(const except::Exception& ex) } } -mt::ThreadGroup::ThreadGroupRunnable::ThreadGroupRunnable - (std::auto_ptr runnable, mt::ThreadGroup& parentThreadGroup): - mRunnable(runnable), mParentThreadGroup(parentThreadGroup) +std::auto_ptr ThreadGroup::getNextInitializer() +{ + std::auto_ptr threadInit(NULL); + if (mAffinityInit.get()) + { + threadInit = mAffinityInit->newThreadInitializer(); + } + + return threadInit; +} + +ThreadGroup::ThreadGroupRunnable::ThreadGroupRunnable( + std::auto_ptr runnable, + ThreadGroup& parentThreadGroup, + std::auto_ptr threadInit) : + mRunnable(runnable), + mParentThreadGroup(parentThreadGroup), + mCPUInit(threadInit) { } -void mt::ThreadGroup::ThreadGroupRunnable::run() +void ThreadGroup::ThreadGroupRunnable::run() { try { + if (mCPUInit.get()) + { + mCPUInit->initialize(); + } mRunnable->run(); } catch(const except::Exception& ex) @@ -123,3 +160,19 @@ void mt::ThreadGroup::ThreadGroupRunnable::run() except::Exception(Ctxt("Unknown ThreadGroup exception."))); } } + +bool ThreadGroup::isPinToCPUEnabled() const +{ + return mAffinityInit.get() != NULL; +} + +bool ThreadGroup::getDefaultPinToCPU() +{ + return DEFAULT_PIN_TO_CPU; +} + +void ThreadGroup::setDefaultPinToCPU(bool newDefault) +{ + DEFAULT_PIN_TO_CPU = newDefault; +} +} diff --git a/externals/coda-oss/modules/c++/mt/tests/ThreadGroupAffinityTest.cpp b/externals/coda-oss/modules/c++/mt/tests/ThreadGroupAffinityTest.cpp new file mode 100644 index 000000000..109cf8d4b --- /dev/null +++ b/externals/coda-oss/modules/c++/mt/tests/ThreadGroupAffinityTest.cpp @@ -0,0 +1,183 @@ +/* ========================================================================= + * This file is part of mt-c++ + * ========================================================================= + * + * (C) Copyright 2004 - 2019, MDA Information Systems LLC + * + * mt-c++ is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this program; If not, + * see . + * + */ + +#if !defined(__APPLE_CC__) && (defined(__linux) || defined(__linux__)) +#include + +#include +#include +#include +#include +using namespace sys; +using namespace mt; + +namespace +{ +class MyRunTask : public Runnable +{ +public: + + MyRunTask(size_t threadNum, sys::AtomicCounter& counter) : + mThread(threadNum), + mThreadCounter(counter) + { + } + + virtual void run() + { + // Print diagnostics from inside the thread + while(true) + { + if (mThread == mThreadCounter.get()) + { + mt::CriticalSection obtainLock(&mMutex); + std::cout << "Thread " << mThread + << " " << sys::ScopedCPUAffinityUnix().toString() + << "\n"; + mThreadCounter.increment(); + break; + } + } + + // Do work, with periodic breaks to give the OS + // the opportunity to change which CPU the thread is + // assigned to + for (size_t trials = 0; trials < 10; ++trials) + { + volatile size_t count = 10000000000; + volatile size_t sum = 0; + for (size_t ii = 0; ii < count; ++ii) + { + sum++; + } + sleep(5); + } + } + +private: + size_t mThread; + sys::Mutex mMutex; + sys::AtomicCounter& mThreadCounter; +}; +} + +int main(int argc, char** argv) +{ + // This program simulates a workload with periodic breaks + // in an attempt to illustrate pinning threads spawned + // by a ThreadGroup to unique CPUs. + // + // It is recommended that one runs the following (or equivalent) + // to view per-process utilization: + // mpstat -P ALL 2 1000 + // + // Running without the --pin option will not perform any pinning. + // Using mpstat, you should be able to see the assignments + // drift over time (if numThreads < numCPUsAvailable). If the + // machine is fairly quiet, drifting may take some time to manifest. + // + // With --pin enabled, there should be no drift -- threads will be + // stuck on the CPUs that are masked in the per-thread diagnostic + // output. + try + { + const size_t numCPUsAvailable = sys::OS().getNumCPUsAvailable(); + + //----------------------------------------------------- + // Handle CLI parameters + //----------------------------------------------------- + cli::ArgumentParser parser; + parser.addArgument("--threads", + "Number of threads to use", + cli::STORE, + "threads", + "INT")->setDefault(numCPUsAvailable); + + parser.addArgument("--pin", + "Enable CPU pinning", + cli::STORE_TRUE, + "pinToCPU")->setDefault(false); + const std::auto_ptr options(parser.parse(argc, argv)); + + const bool pinToCPU = options->get("pinToCPU"); + const size_t numThreads = options->get("threads"); + + //----------------------------------------------------- + // Print diagnostics + //----------------------------------------------------- + std::cout << "Num CPUs available: " << numCPUsAvailable << std::endl; + std::cout << "Num threads requested: " << numThreads << std::endl; + std::cout << "Use CPU pinning: " << pinToCPU << std::endl; + std::cout << "Available CPU mask: " + << sys::ScopedCPUAffinityUnix().toString() << std::endl; + + if (numThreads > numCPUsAvailable && pinToCPU) + { + throw except::Exception( + Ctxt("Requested more threads than CPUs with pinning enabled")); + } + + //----------------------------------------------------- + // Run the thread group operations, + // pinning if requested + //----------------------------------------------------- + ThreadGroup threads(pinToCPU); + const ThreadPlanner planner(numThreads, numThreads); + size_t threadNum = 0; + size_t startElement = 0; + size_t numElementsThisThread = 0; + sys::AtomicCounter threadCounter; + + while(planner.getThreadInfo(threadNum, startElement, numElementsThisThread)) + { + threads.createThread(new MyRunTask(threadNum, threadCounter)); + ++threadNum; + } + + threads.joinAll(); + } + + catch (const except::Throwable& t) + { + std::cout << "Exception Caught: " << t.toString() << std::endl; + return -1; + } + catch (...) + { + std::cout << "Exception Caught!" << std::endl; + return -1; + } + + return 0; +} + +#else + +#include +int main (int, char**) +{ + std::cout << "Usable only on *nix systems" << std::endl; + return 0; +} + +#endif + diff --git a/externals/coda-oss/modules/c++/mt/unittests/ThreadGroupTest.cpp b/externals/coda-oss/modules/c++/mt/unittests/ThreadGroupTest.cpp index 480688c02..9e8f21bf1 100644 --- a/externals/coda-oss/modules/c++/mt/unittests/ThreadGroupTest.cpp +++ b/externals/coda-oss/modules/c++/mt/unittests/ThreadGroupTest.cpp @@ -80,9 +80,35 @@ TEST_CASE(ThreadGroupTest) TEST_ASSERT_EQ(numDeleted, 3); } +TEST_CASE(PinToCPUTest) +{ + bool defaultValue; +#if defined(MT_DEFAULT_PINNING) + defaultValue = true; +#else + defaultValue = false; +#endif + // Check the pinning settings for the default value + TEST_ASSERT_EQ(ThreadGroup::getDefaultPinToCPU(), defaultValue); + ThreadGroup threads1; + TEST_ASSERT_EQ(threads1.isPinToCPUEnabled(), defaultValue); + + // Check the pinning settings when pinning is enabled + ThreadGroup::setDefaultPinToCPU(true); + TEST_ASSERT_EQ(ThreadGroup::getDefaultPinToCPU(), true); + ThreadGroup threads2; + TEST_ASSERT_EQ(threads2.isPinToCPUEnabled(), true); + + // Check the pinning settings when pinning is disabled + ThreadGroup::setDefaultPinToCPU(false); + TEST_ASSERT_EQ(ThreadGroup::getDefaultPinToCPU(), false); + ThreadGroup threads3; + TEST_ASSERT_EQ(threads3.isPinToCPUEnabled(), false); +} + int main(int, char**) { TEST_CHECK(ThreadGroupTest); - + TEST_CHECK(PinToCPUTest); return 0; } diff --git a/externals/coda-oss/modules/c++/mt/wscript b/externals/coda-oss/modules/c++/mt/wscript index 64d218332..271271a42 100644 --- a/externals/coda-oss/modules/c++/mt/wscript +++ b/externals/coda-oss/modules/c++/mt/wscript @@ -2,8 +2,25 @@ NAME = 'mt' MAINTAINER = 'jmrandol@users.sourceforge.net' VERSION = '1.1' MODULE_DEPS = 'sys except math mem types' +TEST_DEPS = 'cli' -options = configure = distclean = lambda p: None +distclean = lambda p: None + +def options(opt): + opt.add_option('--default-cpu-pinning', + action='store_true', + dest='mt_default_cpu_pinning', + default=False, + help='Use affinity-based CPU pinning by default in MT') + +def configure(conf): + from build import writeConfig + from waflib import Options + + def pinning_callback(conf): + if Options.options.mt_default_cpu_pinning: + conf.define('MT_DEFAULT_PINNING', 1) + writeConfig(conf, pinning_callback, NAME) def build(bld): bld.module(**globals()) diff --git a/externals/coda-oss/modules/c++/polygon/include/polygon/Intersections.h b/externals/coda-oss/modules/c++/polygon/include/polygon/Intersections.h index 8773f96c3..34da6ab49 100644 --- a/externals/coda-oss/modules/c++/polygon/include/polygon/Intersections.h +++ b/externals/coda-oss/modules/c++/polygon/include/polygon/Intersections.h @@ -82,7 +82,8 @@ class Intersections Intersections(const std::vector >& points, const types::RowCol& dims, types::RowCol offset = - types::RowCol(0, 0)) + types::RowCol(0, 0)) : + mDims(dims) { if (!points.empty()) { @@ -97,11 +98,13 @@ class Intersections */ void get(size_t row, std::vector& intersections) const { + // Clear out any old intersections in the vector we're given + intersections.clear(); + if (row >= mIntersections.size()) { // Could be because of no points or this is a row greater than // we computed for - intersections.clear(); return; } @@ -109,22 +112,37 @@ class Intersections if (interRow.empty() || interRow.size() % 2 != 0) { // No intersections on this row - intersections.clear(); return; } const size_t numPairs = interRow.size() / 2; - intersections.resize(numPairs); for (size_t pair = 0, idx = 0; pair < numPairs; ++pair) { - Intersection& intersection(intersections[pair]); - const double first = static_cast(interRow[idx++]); - const double last = static_cast(interRow[idx++]); + double first = static_cast(interRow[idx++]); + double last = static_cast(interRow[idx++]); + // If the pair of intersections lies outside of the image, + // there is no intersection for this pair. + const double lastCol = mDims.col - 1; + if ((first < 0.0 && last < 0.0) || + (first > lastCol && last > lastCol)) + { + continue; + } + + // Clamp the intersections to the image boundary + first = std::max(0.0, std::min(lastCol, first)); + last = std::max(0.0, std::min(lastCol, last)); + + Intersection intersection; intersection.first = static_cast(std::ceil(first)); intersection.last = static_cast(std::floor(last)); - if (intersection.last < intersection.first) + if(intersection.last > intersection.first) + { + intersections.push_back(intersection); + } + else { if (first < last) { @@ -132,13 +150,7 @@ class Intersections // Then intersection.first = 56, intersection.last = 55 // We should count 55 as an intersection intersection.first = intersection.last; - } - else - { - // Likely due to some weird rounding issues - // Counts as no intersections - intersections.clear(); - return; + intersections.push_back(intersection); } } } @@ -238,18 +250,14 @@ class Intersections const PointT dcdr = (c1 - c0) / (r1 - r0); // Calculate intersections + // We do not clamp to the image dimensions at this time -- this is + // done inside get(), and allows us to determine if intersection + // pairs lie fully outside of the image. Clamping here ambiguates + // that situation. for (sys::SSize_T row = sl0; row <= sl1; ++row) { const PointT delt = row - r0; - PointT sli = c0 + delt * dcdr; - if (sli < 0) - { - sli = 0; - } - else if (static_cast(sli) >= dims.col) - { - sli = dims.col - 1; - } + const PointT sli = c0 + delt * dcdr; mIntersections[row].push_back(sli); } } @@ -267,6 +275,7 @@ class Intersections } private: + const types::RowCol mDims; std::vector > mIntersections; }; } diff --git a/externals/coda-oss/modules/c++/polygon/unittests/test_polygon_mask.cpp b/externals/coda-oss/modules/c++/polygon/unittests/test_polygon_mask.cpp index b4f630877..78481c4ac 100644 --- a/externals/coda-oss/modules/c++/polygon/unittests/test_polygon_mask.cpp +++ b/externals/coda-oss/modules/c++/polygon/unittests/test_polygon_mask.cpp @@ -160,11 +160,13 @@ TEST_CASE(testWithAllFullRanges) TEST_CASE(testWithAllEmptyRanges) { + // Test for a polygon that is fully outside of the chip, + // but shares rows with the chip std::vector > points; - points.push_back(types::RowCol(2100, -100)); - points.push_back(types::RowCol(2100, 1100)); - points.push_back(types::RowCol(3100, 1100)); - points.push_back(types::RowCol(3100, -100)); + points.push_back(types::RowCol(-1000, -1100)); + points.push_back(types::RowCol(-1000, -100)); + points.push_back(types::RowCol(2000, -100)); + points.push_back(types::RowCol(2000, -1100)); // Draw a mask const types::RowCol dims(1000, 800); @@ -172,6 +174,114 @@ TEST_CASE(testWithAllEmptyRanges) TEST_ASSERT_EQ(mask.getMarkMode(), polygon::PolygonMask::MARK_ALL_FALSE); } + +TEST_CASE(testWithPartialCutBotomLeft) +{ + // Test for a polygon that is partially outside of the chip, clipping + // through only some rows/cols in the bottom-left. + // The line (1, -1) -> (6, 5) cuts diagnonally across the lower-left + std::vector > points; + points.push_back(types::RowCol(1.0, -1.0)); + points.push_back(types::RowCol(6.0, 5.0)); + points.push_back(types::RowCol(6.0, -2.0)); + points.push_back(types::RowCol(3.0, -3.0)); + + // Draw a mask + const types::RowCol dims(6, 7); + const polygon::PolygonMask mask(points, dims); + + // First two rows of the image have no masked columns + TEST_ASSERT_TRUE(mask.getRange(0).empty()); + TEST_ASSERT_TRUE(mask.getRange(1).empty()); + + // Remaining rows have non-empty column masks + TEST_ASSERT_EQ(mask.getRange(2).mStartElement, 0) + TEST_ASSERT_EQ(mask.getRange(2).mNumElements, 1) + + TEST_ASSERT_EQ(mask.getRange(3).mStartElement, 0) + TEST_ASSERT_EQ(mask.getRange(3).mNumElements, 2) + + TEST_ASSERT_EQ(mask.getRange(4).mStartElement, 0) + TEST_ASSERT_EQ(mask.getRange(4).mNumElements, 3) + + TEST_ASSERT_EQ(mask.getRange(5).mStartElement, 0) + TEST_ASSERT_EQ(mask.getRange(5).mNumElements, 4) +} + +TEST_CASE(testWithPartialCutTopRight) +{ + // Test for a polygon that is partially outside of the chip, clipping + // through only some rows/cols in the bottom-left. + // The line (-1, 0) -> (4, 7) cuts diagnonally across the top-right + std::vector > points; + points.push_back(types::RowCol(-1.0, 0.0)); + points.push_back(types::RowCol(4.0, 7.0)); + points.push_back(types::RowCol(2.0, 9.0)); + points.push_back(types::RowCol(-1.0, 9.0)); + + // Draw a mask + const types::RowCol dims(6, 7); + const polygon::PolygonMask mask(points, dims); + + // First 4 rows have non-empty column masks + TEST_ASSERT_EQ(mask.getRange(0).mStartElement, 2) + TEST_ASSERT_EQ(mask.getRange(0).mNumElements, 5) + + TEST_ASSERT_EQ(mask.getRange(1).mStartElement, 3) + TEST_ASSERT_EQ(mask.getRange(1).mNumElements, 4) + + TEST_ASSERT_EQ(mask.getRange(2).mStartElement, 5) + TEST_ASSERT_EQ(mask.getRange(2).mNumElements, 2) + + TEST_ASSERT_EQ(mask.getRange(3).mStartElement, 6) + TEST_ASSERT_EQ(mask.getRange(3).mNumElements, 1) + + // Last two rows of the image have no masked columns + TEST_ASSERT_TRUE(mask.getRange(4).empty()); + TEST_ASSERT_TRUE(mask.getRange(5).empty()); +} + +TEST_CASE(testWithNarrowPassthrough) +{ + // Test for a polygon that passes through the chip, leaving some rows + // unmasked + // + // +--------------+ + // +----|____ | + // | | ------________. + // +----|____ | | + // | ------________+ + // | | + // +--------------+ + std::vector > points; + points.push_back(types::RowCol(0.5, -0.1)); + points.push_back(types::RowCol(1.5, 6.1)); + points.push_back(types::RowCol(3.5, 6.1)); + points.push_back(types::RowCol(2.5, -0.1)); + + // Draw a mask + const types::RowCol dims(6, 7); + const polygon::PolygonMask mask(points, dims); + + // First row is empty + TEST_ASSERT_TRUE(mask.getRange(0).empty()); + + // Row 1 has columns on the left side of the chip + TEST_ASSERT_EQ(mask.getRange(1).mStartElement, 0) + TEST_ASSERT_EQ(mask.getRange(1).mNumElements, 3) + + // Row 2 has all columns + TEST_ASSERT_EQ(mask.getRange(2).mStartElement, 0) + TEST_ASSERT_EQ(mask.getRange(2).mNumElements, 7) + + // Row 3 has columns on the right side of the chip + TEST_ASSERT_EQ(mask.getRange(3).mStartElement, 3) + TEST_ASSERT_EQ(mask.getRange(3).mNumElements, 4) + + // Last two rows are empty + TEST_ASSERT_TRUE(mask.getRange(4).empty()); + TEST_ASSERT_TRUE(mask.getRange(5).empty()); +} } int main(int /*argc*/, char** /*argv*/) @@ -182,5 +292,8 @@ int main(int /*argc*/, char** /*argv*/) TEST_CHECK(testWithMask); TEST_CHECK(testWithAllFullRanges); TEST_CHECK(testWithAllEmptyRanges); + TEST_CHECK(testWithPartialCutBotomLeft); + TEST_CHECK(testWithPartialCutTopRight); + TEST_CHECK(testWithNarrowPassthrough); return 0; } diff --git a/externals/coda-oss/modules/c++/str/source/Convert.cpp b/externals/coda-oss/modules/c++/str/source/Convert.cpp index 202badf89..55f821049 100644 --- a/externals/coda-oss/modules/c++/str/source/Convert.cpp +++ b/externals/coda-oss/modules/c++/str/source/Convert.cpp @@ -1,7 +1,7 @@ /* ========================================================================= - * This file is part of str-c++ + * This file is part of str-c++ * ========================================================================= - * + * * (C) Copyright 2004 - 2014, MDA Information Systems LLC * * str-c++ is free software; you can redistribute it and/or modify @@ -14,8 +14,8 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with this program; If not, + * You should have received a copy of the GNU Lesser General Public + * License along with this program; If not, * see . * */ @@ -77,13 +77,13 @@ unsigned long long str::strtoull(const char *str, char **endptr, int base) template<> int str::getPrecision(const float& ) { - return std::numeric_limits::digits10 + 1; + return std::numeric_limits::max_digits10; } template<> int str::getPrecision(const double& ) { - return std::numeric_limits::digits10 + 1; + return std::numeric_limits::max_digits10; } template<> int str::getPrecision(const long double& ) { - return std::numeric_limits::digits10 + 1; + return std::numeric_limits::max_digits10; } diff --git a/externals/coda-oss/modules/c++/sys/include/sys/AbstractOS.h b/externals/coda-oss/modules/c++/sys/include/sys/AbstractOS.h index 8f9f192b7..32b964eec 100644 --- a/externals/coda-oss/modules/c++/sys/include/sys/AbstractOS.h +++ b/externals/coda-oss/modules/c++/sys/include/sys/AbstractOS.h @@ -215,8 +215,49 @@ class AbstractOS virtual std::string getDSOSuffix() const = 0; + /*! + * \return the number of logical CPUs present on the machine + * (includes hyperthreading) + */ virtual size_t getNumCPUs() const = 0; + /*! + * \return the number of logical CPUs available. This will be + * affected by pinning (e.g. start/affinity/numactl/taskset), + * and will always be <= getNumCPUs. + */ + virtual size_t getNumCPUsAvailable() const = 0; + + /*! + * \return the number of physical CPUs present on the machine + * (excludes hyperthreading) + */ + virtual size_t getNumPhysicalCPUs() const = 0; + + /*! + * \return the number of physical CPUs available. This will be + * affected by pinning (e.g. start/affinity/numactl/taskset), + * and will always be <= getNumPhysicalCPUs. + */ + virtual size_t getNumPhysicalCPUsAvailable() const = 0; + + /*! + * Divide the available CPUs (pinned with numactl/taskset/start/affinity) + * into a set of physical CPUs and a set of hyperthreaded CPUs. Note + * that there is no real distinction between CPUs that share a core, + * and the separation here is not unique. However, there will only ever + * be 1 entry per core in the physical CPUs list, while the remainder + * of CPUs present in the core will be assigned to the htCPU list. + * + * \param[out] physicalCPUs List of physical CPUs. Size of + * getNumPhysicalCPUsAvailable(). + * \param[out] htCPUs List of CPUs that share a core with a CPU in + * 'physicalCPUs'. Size of + * getNumCPUsAvailable() - getNumPhysicalCPUsAvailable(). + */ + virtual void getAvailableCPUs(std::vector& physicalCPUs, + std::vector& htCPUs) const = 0; + /*! * Create a symlink, pathnames can be either absolute or relative */ diff --git a/externals/coda-oss/modules/c++/sys/include/sys/AtomicCounter.h b/externals/coda-oss/modules/c++/sys/include/sys/AtomicCounter.h index ebbdb726e..a7f0ed7aa 100644 --- a/externals/coda-oss/modules/c++/sys/include/sys/AtomicCounter.h +++ b/externals/coda-oss/modules/c++/sys/include/sys/AtomicCounter.h @@ -23,7 +23,7 @@ #ifndef __SYS_ATOMIC_COUNTER_H__ #define __SYS_ATOMIC_COUNTER_H__ -#include +#include #if defined( __GNUC__ ) && ( defined( __i386__ ) || defined( __x86_64__ ) ) #include diff --git a/externals/coda-oss/modules/c++/sys/include/sys/ConditionVarPosix.h b/externals/coda-oss/modules/c++/sys/include/sys/ConditionVarPosix.h index 44dd24de8..d92352938 100644 --- a/externals/coda-oss/modules/c++/sys/include/sys/ConditionVarPosix.h +++ b/externals/coda-oss/modules/c++/sys/include/sys/ConditionVarPosix.h @@ -1,7 +1,7 @@ /* ========================================================================= - * This file is part of sys-c++ + * This file is part of sys-c++ * ========================================================================= - * + * * (C) Copyright 2004 - 2014, MDA Information Systems LLC * * sys-c++ is free software; you can redistribute it and/or modify @@ -14,8 +14,8 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with this program; If not, + * You should have received a copy of the GNU Lesser General Public + * License along with this program; If not, * see . * */ @@ -24,7 +24,7 @@ #ifndef __SYS_THREAD_PTHREAD_CONDITION_VARIABLE_H__ #define __SYS_THREAD_PTHREAD_CONDITION_VARIABLE_H__ -#include +#include #if defined(HAVE_PTHREAD_H) @@ -63,7 +63,7 @@ class ConditionVarPosix : public ConditionVarInterface * Drop (release) the lock */ virtual void dropLock(); - + /*! * Signal using pthread_cond_signal */ @@ -72,9 +72,9 @@ class ConditionVarPosix : public ConditionVarInterface /*! * Wait using pthread_cond_wait * - * WARNING: The user is responsible for locking the mutex prior - * to using this method. There will be no check and on - * certain systems, undefined/unfavorable behavior may + * WARNING: The user is responsible for locking the mutex prior + * to using this method. There will be no check and on + * certain systems, undefined/unfavorable behavior may * result. */ virtual void wait(); @@ -82,12 +82,12 @@ class ConditionVarPosix : public ConditionVarInterface /*! * Wait using pthread_cond_timed_wait. I kept this and the above * function separate only to be explicit. - * \param seconds Fraction of a second to wait. + * \param seconds Fraction of a second to wait. * \todo Want a TimeInterval * - * WARNING: The user is responsible for locking the mutex prior - * to using this method. There will be no check and on - * certain systems, undefined/unfavorable behavior may + * WARNING: The user is responsible for locking the mutex prior + * to using this method. There will be no check and on + * certain systems, undefined/unfavorable behavior may * result. */ virtual void wait(double seconds); @@ -96,12 +96,12 @@ class ConditionVarPosix : public ConditionVarInterface * Broadcast (notify all) */ virtual void broadcast(); - + /*! * Returns the native type. */ pthread_cond_t& getNative(); - + /*! * Return the type name. This function is essentially free, * because it is static RTTI. @@ -110,7 +110,7 @@ class ConditionVarPosix : public ConditionVarInterface { return typeid(mNative).name(); } - + private: // This is set if we own the mutex, to make sure it gets deleted. std::auto_ptr mMutexOwned; diff --git a/externals/coda-oss/modules/c++/sys/include/sys/Conf.h b/externals/coda-oss/modules/c++/sys/include/sys/Conf.h index 149d4c895..11f032a21 100644 --- a/externals/coda-oss/modules/c++/sys/include/sys/Conf.h +++ b/externals/coda-oss/modules/c++/sys/include/sys/Conf.h @@ -23,7 +23,7 @@ #ifndef __SYS_CONF_H__ #define __SYS_CONF_H__ -#include +#include #include #if defined (__APPLE_CC__) diff --git a/externals/coda-oss/modules/c++/sys/include/sys/MutexPosix.h b/externals/coda-oss/modules/c++/sys/include/sys/MutexPosix.h index b1e4d8cb2..39e623847 100644 --- a/externals/coda-oss/modules/c++/sys/include/sys/MutexPosix.h +++ b/externals/coda-oss/modules/c++/sys/include/sys/MutexPosix.h @@ -1,7 +1,7 @@ /* ========================================================================= - * This file is part of sys-c++ + * This file is part of sys-c++ * ========================================================================= - * + * * (C) Copyright 2004 - 2014, MDA Information Systems LLC * * sys-c++ is free software; you can redistribute it and/or modify @@ -14,8 +14,8 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with this program; If not, + * You should have received a copy of the GNU Lesser General Public + * License along with this program; If not, * see . * */ @@ -24,7 +24,7 @@ #ifndef __SYS_MUTEX_POSIX_H__ #define __SYS_MUTEX_POSIX_H__ -#include +#include #if defined(HAVE_PTHREAD_H) @@ -55,17 +55,17 @@ class MutexPosix : public MutexInterface * Lock the mutex. */ virtual void lock(); - + /*! * Unlock the mutex. */ virtual void unlock(); - + /*! * Returns the native type. */ pthread_mutex_t& getNative(); - + /*! * Return the type name. This function is essentially free, * because it is static RTTI. @@ -74,7 +74,7 @@ class MutexPosix : public MutexInterface { return typeid(mNative).name(); } - + private: pthread_mutex_t mNative; }; @@ -82,4 +82,3 @@ class MutexPosix : public MutexInterface #endif #endif - diff --git a/externals/coda-oss/modules/c++/sys/include/sys/OSUnix.h b/externals/coda-oss/modules/c++/sys/include/sys/OSUnix.h index 3e16d9e15..7c2dd366a 100644 --- a/externals/coda-oss/modules/c++/sys/include/sys/OSUnix.h +++ b/externals/coda-oss/modules/c++/sys/include/sys/OSUnix.h @@ -161,8 +161,49 @@ class OSUnix : public AbstractOS virtual std::string getDSOSuffix() const; + /*! + * \return the number of logical CPUs present on the machine + * (includes hyperthreading) + */ virtual size_t getNumCPUs() const; + /*! + * \return the number of logical CPUs available. This will be + * affected by pinning (e.g. numactl/taskset), and will + * always be <= getNumCPUs. + */ + virtual size_t getNumCPUsAvailable() const; + + /*! + * \return the number of physical CPUs present on the machine + * (excludes hyperthreading) + */ + virtual size_t getNumPhysicalCPUs() const; + + /*! + * \return the number of physical CPUs available. This will be + * affected by pinning (e.g. numactl/taskset), and will + * always be <= getNumPhysicalCPUs. + */ + virtual size_t getNumPhysicalCPUsAvailable() const; + + /*! + * Divide the available CPUs (pinned with numactl/taskset) into + * a set of physical CPUs and a set of hyperthreaded CPUs. Note + * that there is no real distinction between CPUs that share a core, + * and the separation here is not unique. However, there will only ever + * be 1 entry per core in the physical CPUs list, while the remainder + * of CPUs present in the core will be assigned to the htCPU list. + * + * \param[out] physicalCPUs List of physical CPUs. Size of + * getNumPhysicalCPUsAvailable(). + * \param[out] htCPUs List of CPUs that share a core with a CPU in + * 'physicalCPUs'. Size of + * getNumCPUsAvailable() - getNumPhysicalCPUsAvailable(). + */ + virtual void getAvailableCPUs(std::vector& physicalCPUs, + std::vector& htCPUs) const; + /*! * Create a symlink, pathnames can be either absolute or relative */ diff --git a/externals/coda-oss/modules/c++/sys/include/sys/OSWin32.h b/externals/coda-oss/modules/c++/sys/include/sys/OSWin32.h index 19e34b2e4..bd644ac91 100644 --- a/externals/coda-oss/modules/c++/sys/include/sys/OSWin32.h +++ b/externals/coda-oss/modules/c++/sys/include/sys/OSWin32.h @@ -1,7 +1,7 @@ /* ========================================================================= - * This file is part of sys-c++ + * This file is part of sys-c++ * ========================================================================= - * + * * (C) Copyright 2004 - 2014, MDA Information Systems LLC * * sys-c++ is free software; you can redistribute it and/or modify @@ -14,8 +14,8 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with this program; If not, + * You should have received a copy of the GNU Lesser General Public + * License along with this program; If not, * see . * */ @@ -43,7 +43,7 @@ namespace sys * \class OSWin32 * \brief The abstraction definition layer for windows * - * This class is the abstraction layer as defined for + * This class is the abstraction layer as defined for * the windows operating system. */ class OSWin32 : public AbstractOS @@ -70,7 +70,7 @@ class OSWin32 : public AbstractOS /*! * Determine the username * \return The username - */ + */ //virtual std::string getUsername() const; @@ -89,10 +89,10 @@ class OSWin32 : public AbstractOS * (including its children) either in the same directory or across * directories. The one caveat is that it will fail on directory * moves when the destination is on a different volume. - * + * * \return True upon success, false if failure */ - virtual bool move(const std::string& path, + virtual bool move(const std::string& path, const std::string& newPath) const; /*! @@ -172,7 +172,7 @@ class OSWin32 : public AbstractOS /*! * Set an environment variable */ - virtual void setEnv(const std::string& var, + virtual void setEnv(const std::string& var, const std::string& val, bool overwrite); @@ -181,12 +181,58 @@ class OSWin32 : public AbstractOS */ virtual void unsetEnv(const std::string& var); + /*! + * \return the number of logical CPUs present on the machine + * (includes hyperthreading) + */ virtual size_t getNumCPUs() const; + /*! + * \todo Not yet implemented + * \return the number of logical CPUs available. This will be + * affected by pinning (e.g. start/affinity), and will + * always be <= getNumCPUs. + */ + virtual size_t getNumCPUsAvailable() const; + + /*! + * \todo Not yet implemented + * \return the number of physical CPUs present on the machine + * (excludes hyperthreading) + */ + virtual size_t getNumPhysicalCPUs() const; + + /*! + * \todo Not yet implemented + * \return the number of physical CPUs available. This will be + * affected by pinning (e.g. start/affinity), and will + * always be <= getNumPhysicalCPUs. + */ + virtual size_t getNumPhysicalCPUsAvailable() const; + + /*! + * Divide the available CPUs (pinned with start/affinity) into + * a set of physical CPUs and a set of hyperthreaded CPUs. Note + * that there is no real distinction between CPUs that share a core, + * and the separation here is not unique. However, there will only ever + * be 1 entry per core in the physical CPUs list, while the remainder + * of CPUs present in the core will be assigned to the htCPU list. + * + * \todo Not yet implemented + * + * \param[out] physicalCPUs List of physical CPUs. Size of + * getNumPhysicalCPUsAvailable(). + * \param[out] htCPUs List of CPUs that share a core with a CPU in + * 'physicalCPUs'. Size of + * getNumCPUsAvailable() - getNumPhysicalCPUsAvailable(). + */ + virtual void getAvailableCPUs(std::vector& physicalCPUs, + std::vector& htCPUs) const; + /*! * Create a symlink, pathnames can be either absolute or relative */ - virtual void createSymlink(const std::string& origPathname, + virtual void createSymlink(const std::string& origPathname, const std::string& symlinkPathname) const; /*! diff --git a/externals/coda-oss/modules/c++/sys/include/sys/ScopedCPUAffinityUnix.h b/externals/coda-oss/modules/c++/sys/include/sys/ScopedCPUAffinityUnix.h new file mode 100644 index 000000000..900239244 --- /dev/null +++ b/externals/coda-oss/modules/c++/sys/include/sys/ScopedCPUAffinityUnix.h @@ -0,0 +1,124 @@ +/* ========================================================================= + * This file is part of sys-c++ + * ========================================================================= + * + * (C) Copyright 2004 - 2019, MDA Information Systems LLC + * + * sys-c++ is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this program; If not, + * see . + * + */ + +#ifndef __SYS_SCOPED_CPU_AFFINITY_UNIX_H__ +#define __SYS_SCOPED_CPU_AFFINITY_UNIX_H__ + +#include + +#if !defined(WIN32) + +#include +#include +#include + +namespace sys +{ +/*! + * \class ScopedCPUMaskUnix + * \brief Class for generating a CPU mask on Unix + * + * ScopedCPUMaskUnix handles the creation/destruction of a dynamically + * created cpu_set_t on Unix systems. This allows for the creation of + * CPU masks greater than the number of CPUs reported by CPU_SETSIZE. + */ +class ScopedCPUMaskUnix +{ +public: + /*! + * Constructor which automatically creates a mask that can hold + * at least the number of online CPUs + */ + ScopedCPUMaskUnix(); + + /*! + * Constructor for a given number of CPUs. Note that the resulting + * CPU set may be larger, as the mask must be able to hold at least + * CPU_SETSIZE number of CPUs. + */ + ScopedCPUMaskUnix(int numCPUs); + + /*! + * Destructor + */ + virtual ~ScopedCPUMaskUnix(); + + /*! + * \returns a 0-1 string representation for the CPU mask, + * where 1s represent a usable CPU and 0 an inactive CPU + */ + std::string toString() const; + + //! \returns a const cpu_set_t* mask + const cpu_set_t* getMask() const + { + return mMask; + } + + //! \returns a mutable cpu_set_t* mask + cpu_set_t* getMask() + { + return mMask; + } + + //! \returns the size of the CPU set in bytes + size_t getSize() const + { + return mSize; + } + + //! \returns the number of online CPUs + static int getNumOnlineCPUs(); + +private: + void initialize(int numCPUs); + +protected: + size_t mSize; + cpu_set_t* mMask; +}; + +/*! + * \class ScopedCPUAffinityUnix + * \brief Class for generating a CPU mask on Unix that corresponds + * to the current CPU affinity + * + * ScopedCPUAffinityUnix handles the creation/destruction of a dynamically + * created cpu_set_t on Unix systems that represents the CPU affinity for + * the current PID. + */ +class ScopedCPUAffinityUnix : public ScopedCPUMaskUnix +{ +public: + /*! + * Constructor that obtains the CPU affinity mask for the current + * process. + * + * Note that if called from inside a thread, the CPU affinity for that + * thread is generated. + */ + ScopedCPUAffinityUnix(); +}; +} + +#endif +#endif diff --git a/externals/coda-oss/modules/c++/sys/include/sys/SemaphorePosix.h b/externals/coda-oss/modules/c++/sys/include/sys/SemaphorePosix.h index ea4ccc625..49078abc3 100644 --- a/externals/coda-oss/modules/c++/sys/include/sys/SemaphorePosix.h +++ b/externals/coda-oss/modules/c++/sys/include/sys/SemaphorePosix.h @@ -1,7 +1,7 @@ /* ========================================================================= - * This file is part of sys-c++ + * This file is part of sys-c++ * ========================================================================= - * + * * (C) Copyright 2004 - 2014, MDA Information Systems LLC * * sys-c++ is free software; you can redistribute it and/or modify @@ -14,8 +14,8 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with this program; If not, + * You should have received a copy of the GNU Lesser General Public + * License along with this program; If not, * see . * */ @@ -24,7 +24,7 @@ #ifndef __SYS_SEMAPHORE_POSIX_H__ #define __SYS_SEMAPHORE_POSIX_H__ -#include +#include #if defined(HAVE_PTHREAD_H) && !defined(__APPLE_CC__) @@ -47,7 +47,7 @@ class SemaphorePosix : public SemaphoreInterface void wait(); void signal(); sem_t& getNative(); - + /*! * Return the type name. This function is essentially free, * because it is static RTTI. diff --git a/externals/coda-oss/modules/c++/sys/include/sys/ThreadPosix.h b/externals/coda-oss/modules/c++/sys/include/sys/ThreadPosix.h index 0d46f5498..51e6d5316 100644 --- a/externals/coda-oss/modules/c++/sys/include/sys/ThreadPosix.h +++ b/externals/coda-oss/modules/c++/sys/include/sys/ThreadPosix.h @@ -1,7 +1,7 @@ /* ========================================================================= - * This file is part of sys-c++ + * This file is part of sys-c++ * ========================================================================= - * + * * (C) Copyright 2004 - 2014, MDA Information Systems LLC * * sys-c++ is free software; you can redistribute it and/or modify @@ -14,8 +14,8 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with this program; If not, + * You should have received a copy of the GNU Lesser General Public + * License along with this program; If not, * see . * */ @@ -24,7 +24,7 @@ #ifndef __SYS_THREAD_PTHREAD_THREAD_H__ #define __SYS_THREAD_PTHREAD_THREAD_H__ -#include +#include #if defined(HAVE_PTHREAD_H) @@ -55,8 +55,8 @@ namespace sys /*! * \class ThreadPosix * \brief The implementation of a pthread-specialized thread - * - * This class provides the wrapper for a pthread_t. + * + * This class provides the wrapper for a pthread_t. * */ class ThreadPosix : public ThreadInterface @@ -120,7 +120,7 @@ class ThreadPosix : public ThreadInterface * Calls sched_yield to yield the thread of control */ static void yield(); - + /*! * Returns the native type. You probably should not use this * unless you have specific constraints on which package you use diff --git a/externals/coda-oss/modules/c++/sys/include/sys/config.h b/externals/coda-oss/modules/c++/sys/include/sys/config.h deleted file mode 100644 index e1553e234..000000000 --- a/externals/coda-oss/modules/c++/sys/include/sys/config.h +++ /dev/null @@ -1,137 +0,0 @@ -/* ========================================================================= - * This file is part of sys-c++ - * ========================================================================= - * - * (C) Copyright 2004 - 2014, MDA Information Systems LLC - * - * sys-c++ is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation; either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this program; If not, - * see . - * - */ - - -/* include/sys/config.h. Generated by configure. */ -/* include/sys/config.h.in. Generated from configure.in by autoheader. */ - -/* Define to 1 if using `getloadavg.c'. */ -/* #undef C_GETLOADAVG */ - -/* Define to 1 for DGUX with . */ -/* #undef DGUX */ - -/* Define to 1 if the `getloadavg' function needs to be run setuid or setgid. - */ -/* #undef GETLOADAVG_PRIVILEGED */ - -/* Define to 1 if you don't have `vprintf' but do have `_doprnt.' */ -/* #undef HAVE_DOPRNT */ - -/* Define to 1 if you have the `getloadavg' function. */ -/* #undef HAVE_GETLOADAVG */ - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_INTTYPES_H */ - -/* Define to 1 if you have the `' library (-l). */ -/* #undef HAVE_LIB */ - -/* Define to 1 if you have the `dgc' library (-ldgc). */ -/* #undef HAVE_LIBDGC */ - -/* Define to 1 if you have the `kstat' library (-lkstat). */ -/* #undef HAVE_LIBKSTAT */ - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_MACH_MACH_H */ - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_MEMORY_H */ - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_NLIST_H */ - -/* Define to 1 if you have the `pstat_getdynamic' function. */ -/* #undef HAVE_PSTAT_GETDYNAMIC */ - -/* Define to 1 if you have the `setlocale' function. */ -/* #undef HAVE_SETLOCALE */ - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_STDINT_H */ - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_STDLIB_H */ - -/* Define to 1 if you have the `strftime' function. */ -/* #undef HAVE_STRFTIME */ - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_STRINGS_H */ - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_STRING_H */ - -/* Define to 1 if `n_un.n_name' is member of `struct nlist'. */ -/* #undef HAVE_STRUCT_NLIST_N_UN_N_NAME */ - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_SYS_STAT_H */ - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_SYS_TYPES_H */ - -/* Define to 1 if you have that is POSIX.1 compatible. */ -/* #undef HAVE_SYS_WAIT_H */ - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_UNISTD_H */ - -/* Define to 1 if you have the `vprintf' function. */ -/* #undef HAVE_VPRINTF */ - -/* Define to 1 if your `struct nlist' has an `n_un' member. Obsolete, depend - on `HAVE_STRUCT_NLIST_N_UN_N_NAME */ -/* #undef NLIST_NAME_UNION */ - -/* Define to the address where bug reports for this package should be sent. */ -/* #undef PACKAGE_BUGREPORT */ - -/* Define to the full name of this package. */ -/* #undef PACKAGE_NAME */ - -/* Define to the full name and version of this package. */ -/* #undef PACKAGE_STRING */ - -/* Define to the one symbol short name of this package. */ -/* #undef PACKAGE_TARNAME */ - -/* Define to the version of this package. */ -/* #undef PACKAGE_VERSION */ - -/* The size of a `', as computed by sizeof. */ -/* #undef SIZEOF_ */ - -/* Define to 1 if you have the ANSI C header files. */ -/* #undef STDC_HEADERS */ - -/* Define to 1 on System V Release 4. */ -/* #undef SVR4 */ - -/* Define to 1 for Encore UMAX. */ -/* #undef UMAX */ - -/* Define to 1 for Encore UMAX 4.3 that has instead of - . */ -/* #undef UMAX4_3 */ - diff --git a/externals/coda-oss/modules/c++/sys/include/sys/config.h.in b/externals/coda-oss/modules/c++/sys/include/sys/config.h.in deleted file mode 100644 index c3eea857c..000000000 --- a/externals/coda-oss/modules/c++/sys/include/sys/config.h.in +++ /dev/null @@ -1,113 +0,0 @@ -/* include/sys/config.h.in. Generated from configure.in by autoheader. */ - -/* Define to 1 if using `getloadavg.c'. */ -#undef C_GETLOADAVG - -/* Define to 1 for DGUX with . */ -#undef DGUX - -/* Define to 1 if the `getloadavg' function needs to be run setuid or setgid. - */ -#undef GETLOADAVG_PRIVILEGED - -/* Define to 1 if you don't have `vprintf' but do have `_doprnt.' */ -#undef HAVE_DOPRNT - -/* Define to 1 if you have the `getloadavg' function. */ -#undef HAVE_GETLOADAVG - -/* Define to 1 if you have the header file. */ -#undef HAVE_INTTYPES_H - -/* Define to 1 if you have the `' library (-l). */ -#undef HAVE_LIB - -/* Define to 1 if you have the `dgc' library (-ldgc). */ -#undef HAVE_LIBDGC - -/* Define to 1 if you have the `kstat' library (-lkstat). */ -#undef HAVE_LIBKSTAT - -/* Define to 1 if you have the header file. */ -#undef HAVE_MACH_MACH_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_MEMORY_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_NLIST_H - -/* Define to 1 if you have the `pstat_getdynamic' function. */ -#undef HAVE_PSTAT_GETDYNAMIC - -/* Define to 1 if you have the `setlocale' function. */ -#undef HAVE_SETLOCALE - -/* Define to 1 if you have the header file. */ -#undef HAVE_STDINT_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_STDLIB_H - -/* Define to 1 if you have the `strftime' function. */ -#undef HAVE_STRFTIME - -/* Define to 1 if you have the header file. */ -#undef HAVE_STRINGS_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_STRING_H - -/* Define to 1 if `n_un.n_name' is member of `struct nlist'. */ -#undef HAVE_STRUCT_NLIST_N_UN_N_NAME - -/* Define to 1 if you have the header file. */ -#undef HAVE_SYS_STAT_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_SYS_TYPES_H - -/* Define to 1 if you have that is POSIX.1 compatible. */ -#undef HAVE_SYS_WAIT_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_UNISTD_H - -/* Define to 1 if you have the `vprintf' function. */ -#undef HAVE_VPRINTF - -/* Define to 1 if your `struct nlist' has an `n_un' member. Obsolete, depend - on `HAVE_STRUCT_NLIST_N_UN_N_NAME */ -#undef NLIST_NAME_UNION - -/* Define to the address where bug reports for this package should be sent. */ -#undef PACKAGE_BUGREPORT - -/* Define to the full name of this package. */ -#undef PACKAGE_NAME - -/* Define to the full name and version of this package. */ -#undef PACKAGE_STRING - -/* Define to the one symbol short name of this package. */ -#undef PACKAGE_TARNAME - -/* Define to the version of this package. */ -#undef PACKAGE_VERSION - -/* The size of a `', as computed by sizeof. */ -#undef SIZEOF_ - -/* Define to 1 if you have the ANSI C header files. */ -#undef STDC_HEADERS - -/* Define to 1 on System V Release 4. */ -#undef SVR4 - -/* Define to 1 for Encore UMAX. */ -#undef UMAX - -/* Define to 1 for Encore UMAX 4.3 that has instead of - . */ -#undef UMAX4_3 - diff --git a/externals/coda-oss/modules/c++/sys/source/Backtrace.cpp b/externals/coda-oss/modules/c++/sys/source/Backtrace.cpp index 184b7a3c8..212c988f9 100644 --- a/externals/coda-oss/modules/c++/sys/source/Backtrace.cpp +++ b/externals/coda-oss/modules/c++/sys/source/Backtrace.cpp @@ -1,7 +1,7 @@ /* ========================================================================= * This file is part of sys-c++ * ========================================================================= - * + * * (C) Copyright 2004 - 2016, MDA Information Systems LLC * * sys-c++ is free software; you can redistribute it and/or modify @@ -14,15 +14,15 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with this program; If not, + * You should have received a copy of the GNU Lesser General Public + * License along with this program; If not, * see . * */ #include -#include +#include #include static const size_t MAX_STACK_ENTRIES = 62; @@ -62,7 +62,7 @@ std::string sys::getBacktrace() { void* stackBuffer[MAX_STACK_ENTRIES]; int currentStackSize = backtrace(stackBuffer, MAX_STACK_ENTRIES); - BacktraceHelper stackSymbols(backtrace_symbols(stackBuffer, + BacktraceHelper stackSymbols(backtrace_symbols(stackBuffer, currentStackSize)); std::stringstream ss; @@ -83,4 +83,3 @@ std::string sys::getBacktrace() } #endif - diff --git a/externals/coda-oss/modules/c++/sys/source/DateTime.cpp b/externals/coda-oss/modules/c++/sys/source/DateTime.cpp index 60dada55c..2346b2091 100644 --- a/externals/coda-oss/modules/c++/sys/source/DateTime.cpp +++ b/externals/coda-oss/modules/c++/sys/source/DateTime.cpp @@ -1,7 +1,7 @@ /* ========================================================================= - * This file is part of sys-c++ + * This file is part of sys-c++ * ========================================================================= - * + * * (C) Copyright 2004 - 2014, MDA Information Systems LLC * * sys-c++ is free software; you can redistribute it and/or modify @@ -14,13 +14,13 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with this program; If not, + * You should have received a copy of the GNU Lesser General Public + * License along with this program; If not, * see . * */ -#include +#include #include "except/Exception.h" #include "sys/DateTime.h" @@ -121,7 +121,7 @@ char* strptime(const char *buf, const char *fmt, struct tm& tm, double& millis) "Value does not match format (%%): " + bc)); break; - /* + /* * "Complex" conversion rules, implemented through recursion. */ case 'c': // Date and time, using the locale's format. @@ -294,7 +294,7 @@ char* strptime(const char *buf, const char *fmt, struct tm& tm, double& millis) case 'U': // The week of year, beginning on sunday. case 'W': // The week of year, beginning on monday. - /* + /* * XXX This is bogus, as we can not assume any valid * information present in the tm structure at this * point to calculate a real value, so just check the @@ -586,8 +586,8 @@ void sys::DateTime::setTime(const std::string& time, const std::string& format) std::string sys::DateTime::format(const std::string& formatStr) const { - // the longest string expansion is - // %c => 'Thu Aug 23 14:55:02 2001' + // the longest string expansion is + // %c => 'Thu Aug 23 14:55:02 2001' // which is an expansion of 22 characters size_t maxSize = formatStr.length() * 22 + 1; std::vector expanded(maxSize); diff --git a/externals/coda-oss/modules/c++/sys/source/LocalDateTime.cpp b/externals/coda-oss/modules/c++/sys/source/LocalDateTime.cpp index c66324c95..20e46894d 100755 --- a/externals/coda-oss/modules/c++/sys/source/LocalDateTime.cpp +++ b/externals/coda-oss/modules/c++/sys/source/LocalDateTime.cpp @@ -1,7 +1,7 @@ /* ========================================================================= - * This file is part of sys-c++ + * This file is part of sys-c++ * ========================================================================= - * + * * (C) Copyright 2004 - 2014, MDA Information Systems LLC * * sys-c++ is free software; you can redistribute it and/or modify @@ -14,15 +14,15 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with this program; If not, + * You should have received a copy of the GNU Lesser General Public + * License along with this program; If not, * see . * */ #include -#include +#include #include diff --git a/externals/coda-oss/modules/c++/sys/source/OSUnix.cpp b/externals/coda-oss/modules/c++/sys/source/OSUnix.cpp index 4860b859a..90e2966a8 100644 --- a/externals/coda-oss/modules/c++/sys/source/OSUnix.cpp +++ b/externals/coda-oss/modules/c++/sys/source/OSUnix.cpp @@ -20,7 +20,7 @@ * */ -#include "sys/sys_config.h" +#include "config/coda_oss_config.h" #if !defined(WIN32) @@ -29,6 +29,10 @@ #include #include #include +#include +#include +#include +#include #if defined(__APPLE__) @@ -43,6 +47,8 @@ #include "sys/OSUnix.h" #include "sys/File.h" +#include "sys/ScopedCPUAffinityUnix.h" +#include "str/Tokenizer.h" namespace @@ -80,6 +86,52 @@ class CharWrapper private: char* mArray; }; + +std::set get_unique_thread_siblings() +{ + // Our goal is to count the number of unique entries that occur + // in the files /sys/devices/system/cpu/cpu*/topology/thread_siblings_list + const sys::Path sysCPUPath("/sys/devices/system/cpu"); + if (!sysCPUPath.isDirectory()) + { + throw except::Exception( + Ctxt("Expected dir /sys/devices/system/cpu does not exist")); + } + + const std::vector searchPaths(1, sysCPUPath.getPath()); + const std::vector subDirs = + sys::FileFinder::search( + sys::DirectoryOnlyPredicate(), + searchPaths, + false); + + std::set unique_ts; + for (std::vector::const_iterator ii = subDirs.begin(); + ii != subDirs.end(); + ++ii) + { + const sys::Path tsPath(*ii, "topology/thread_siblings_list"); + if (tsPath.exists()) + { + std::ifstream tsIFS(tsPath.getPath().c_str()); + if (!tsIFS.is_open()) + { + std::ostringstream msg; + msg << "Unable to open thread siblings file " + << tsPath.getPath(); + throw except::Exception(Ctxt(msg.str())); + } + + std::string tsContents; + tsIFS >> tsContents; + tsIFS.close(); + + unique_ts.insert(tsContents); + } + } + + return unique_ts; +} } std::string sys::OSUnix::getPlatformName() const @@ -302,11 +354,67 @@ void sys::OSUnix::unsetEnv(const std::string& var) size_t sys::OSUnix::getNumCPUs() const { -#ifdef _SC_NPROCESSORS_ONLN - return sysconf(_SC_NPROCESSORS_ONLN); -#else - throw except::NotImplementedException(Ctxt("Unable to get the number of CPUs")); -#endif + return static_cast(ScopedCPUMaskUnix::getNumOnlineCPUs()); +} + +size_t sys::OSUnix::getNumCPUsAvailable() const +{ + const ScopedCPUAffinityUnix mask; + return CPU_COUNT_S(mask.getSize(), mask.getMask()); +} + +size_t sys::OSUnix::getNumPhysicalCPUs() const +{ + return get_unique_thread_siblings().size(); +} + +size_t sys::OSUnix::getNumPhysicalCPUsAvailable() const +{ + std::vector physicalCPUs; + std::vector htCPUs; + getAvailableCPUs(physicalCPUs, htCPUs); + return physicalCPUs.size(); +} + +void sys::OSUnix::getAvailableCPUs(std::vector& physicalCPUs, + std::vector& htCPUs) const +{ + physicalCPUs.clear(); + htCPUs.clear(); + + // Obtain scheduling affinity for all CPUs (including hyperthreading) + const ScopedCPUAffinityUnix mask; + + // Cross-reference the thread siblings with active CPUs + // and separate into physical CPUs and HT CPUs. At the hardware level there + // is no distinction as to which CPU is the physical and which is the HT. + // So, we'll just say that the first masked CPU ID encountered for a core + // is the physical CPU, and the remainder of masked CPU IDs are the HT CPUs. + const std::set unique_ts(get_unique_thread_siblings()); + std::set::const_iterator tsStr; + for (tsStr = unique_ts.begin(); tsStr != unique_ts.end(); ++tsStr) + { + bool foundPhysical = false; + const str::Tokenizer::Tokens cpuIDs = str::Tokenizer(*tsStr, ","); + for (str::Tokenizer::Tokens::const_iterator cpu = cpuIDs.begin(); + cpu != cpuIDs.end(); + ++cpu) + { + const int cpuInt = str::toType(*cpu); + if (CPU_ISSET_S(cpuInt, mask.getSize(), mask.getMask())) + { + if (foundPhysical) + { + htCPUs.push_back(cpuInt); + } + else + { + physicalCPUs.push_back(cpuInt); + foundPhysical = true; + } + } + } + } } void sys::OSUnix::createSymlink(const std::string& origPathname, diff --git a/externals/coda-oss/modules/c++/sys/source/OSWin32.cpp b/externals/coda-oss/modules/c++/sys/source/OSWin32.cpp index 43bd26c27..7468bfa1f 100644 --- a/externals/coda-oss/modules/c++/sys/source/OSWin32.cpp +++ b/externals/coda-oss/modules/c++/sys/source/OSWin32.cpp @@ -281,6 +281,34 @@ size_t sys::OSWin32::getNumCPUs() const return info.dwNumberOfProcessors; } +size_t sys::OSWin32::getNumCPUsAvailable() const +{ + throw except::NotImplementedException( + Ctxt("Windows getNumCPUsAvailable not yet implemented.")); +} + +size_t sys::OSWin32::getNumPhysicalCPUs() const +{ + // TODO Need to use GetLogicalProcessorInformationEx. + // See reference implementation at + // https://devblogs.microsoft.com/oldnewthing/?p=2823 + throw except::NotImplementedException( + Ctxt("Windows getNumPhysicalCPUs not yet implemented.")); +} + +size_t sys::OSWin32::getNumPhysicalCPUsAvailable() const +{ + throw except::NotImplementedException( + Ctxt("Windows getNumPhysicalCPUsAvailable not yet implemented.")); +} + +void sys::OSWin32::getAvailableCPUs(std::vector& /*physicalCPUs*/, + std::vector& /*htCPUs*/) const +{ + throw except::NotImplementedException( + Ctxt("Windows getAvailableCPUs not yet implemented.")); +} + void sys::OSWin32::createSymlink(const std::string& origPathname, const std::string& symlinkPathname) const { diff --git a/externals/coda-oss/modules/c++/sys/source/ScopedCPUAffinityUnix.cpp b/externals/coda-oss/modules/c++/sys/source/ScopedCPUAffinityUnix.cpp new file mode 100644 index 000000000..e907ac17f --- /dev/null +++ b/externals/coda-oss/modules/c++/sys/source/ScopedCPUAffinityUnix.cpp @@ -0,0 +1,131 @@ +/* ========================================================================= + * This file is part of sys-c++ + * ========================================================================= + * + * (C) Copyright 2004 - 2019, MDA Information Systems LLC + * + * sys-c++ is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this program; If not, + * see . + * + */ + +#include "config/coda_oss_config.h" + +#if !defined(WIN32) + +#include + +#include +#include + +#include + +namespace sys +{ +ScopedCPUMaskUnix::ScopedCPUMaskUnix() +{ + // The number of processors in the mask must be _at least_ the number of + // CPU's representable by the kernel's internal mask. This is given by + // the constant CPU_SETSIZE. + const int numOnlineCPUs = ScopedCPUAffinityUnix::getNumOnlineCPUs(); + const int maxCPUs = std::max(numOnlineCPUs, CPU_SETSIZE); + initialize(maxCPUs); +} + +ScopedCPUMaskUnix::ScopedCPUMaskUnix(int numCPUs) +{ + initialize(numCPUs); +} + +void ScopedCPUMaskUnix::initialize(int numCPUs) +{ + mSize = CPU_ALLOC_SIZE(numCPUs); + mMask = CPU_ALLOC(numCPUs); + + if (mMask == NULL) + { + std::ostringstream msg; + msg << "Failed to allocate CPU mask for " << numCPUs << "CPUs"; + throw except::Exception(Ctxt(msg.str())); + } + + CPU_ZERO_S(mSize, mMask); +} + +ScopedCPUMaskUnix::~ScopedCPUMaskUnix() +{ + if (mMask != NULL) + { + CPU_FREE(mMask); + } +} + +std::string ScopedCPUMaskUnix::toString() const +{ + std::ostringstream str; + const int numCPUs = getNumOnlineCPUs(); + for (int cpu = 0; cpu < numCPUs; ++cpu) + { + str << (CPU_ISSET_S(cpu, mSize, mMask) ? "1" : "0"); + } + + return str.str(); +} + +int ScopedCPUMaskUnix::getNumOnlineCPUs() +{ +#ifdef _SC_NPROCESSORS_ONLN + const int numOnlineCPUs = sysconf(_SC_NPROCESSORS_ONLN); + if (numOnlineCPUs == -1) + { + throw except::Exception(Ctxt("Failed to get online CPU count")); + } + return numOnlineCPUs; +#else +throw except::NotImplementedException(Ctxt("Unable to get the number of CPUs")); +#endif +} + +//----------------------------------------------------------------------------- + +ScopedCPUAffinityUnix::ScopedCPUAffinityUnix() : + ScopedCPUMaskUnix() +{ + if (sched_getaffinity(0, mSize, mMask) == -1) + { + std::ostringstream msg; + msg << "Failed to get CPU affinity with" + << " CPU_SETSIZE=" << CPU_SETSIZE << "," + << " numOnlineCPUs=" << getNumOnlineCPUs() << "," + << " alloc size=" << mSize << "."; + switch (errno) + { + case EINVAL: + msg << " Affinity mask smaller than kernel mask size."; + break; + case EFAULT: + msg << " Invalid mask address."; + break; + case ESRCH: + msg << " PID for current process is invalid."; + break; + default: + msg << " Unknown cause."; + } + throw except::Exception(Ctxt(msg.str())); + } +} +} + +#endif diff --git a/externals/coda-oss/modules/c++/sys/source/UTCDateTime.cpp b/externals/coda-oss/modules/c++/sys/source/UTCDateTime.cpp index 16c931acd..e6c7dda4f 100755 --- a/externals/coda-oss/modules/c++/sys/source/UTCDateTime.cpp +++ b/externals/coda-oss/modules/c++/sys/source/UTCDateTime.cpp @@ -1,7 +1,7 @@ /* ========================================================================= - * This file is part of sys-c++ + * This file is part of sys-c++ * ========================================================================= - * + * * (C) Copyright 2004 - 2014, MDA Information Systems LLC * * sys-c++ is free software; you can redistribute it and/or modify @@ -14,15 +14,15 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * - * You should have received a copy of the GNU Lesser General Public - * License along with this program; If not, + * You should have received a copy of the GNU Lesser General Public + * License along with this program; If not, * see . * */ #include -#include +#include #include #include #include @@ -121,7 +121,7 @@ void UTCDateTime::toMillis() mTimeInMillis = (mSecond + mMinute * SECS_IN_MIN + mHour * SECS_IN_HOUR + numDaysSinceEpoch * SECS_IN_DAY) * 1000.0; mDayOfYear = numDaysThisYear + 1; - + /* January 1, 1970 was a Thursday (5) */ mDayOfWeek = (numDaysSinceEpoch + 5) % 7; } diff --git a/externals/coda-oss/modules/c++/sys/tests/CPUTest.cpp b/externals/coda-oss/modules/c++/sys/tests/CPUTest.cpp new file mode 100644 index 000000000..63df432a5 --- /dev/null +++ b/externals/coda-oss/modules/c++/sys/tests/CPUTest.cpp @@ -0,0 +1,78 @@ +/* ========================================================================= + * This file is part of sys-c++ + * ========================================================================= + * + * (C) Copyright 2004 - 2019, MDA Information Systems LLC + * + * sys-c++ is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this program; If not, + * see . + * + */ +#include + +#include + +namespace +{ +void printCPUs(const std::string& header, const std::vector& cpus) +{ + std::vector::const_iterator cpu; + + std::cout << header << ": "; + for (cpu = cpus.begin(); cpu != cpus.end(); ++cpu) + { + if (cpu != cpus.begin()) + { + std::cout << ", "; + } + std::cout << *cpu; + } + std::cout << std::endl; +} +} + +int main(int /*argc*/, char** /*argv*/) +{ + try + { + sys::OS os; + + std::cout << "Present number of CPUs: " + << os.getNumCPUs() << std::endl; + std::cout << "Present number of physical CPUs: " + << os.getNumPhysicalCPUs() << std::endl; + std::cout << "Available number of CPUs: " + << os.getNumCPUsAvailable() << std::endl; + std::cout << "Available number of physical CPUs: " + << os.getNumPhysicalCPUsAvailable() << std::endl; + + std::vector physicalCPUs; + std::vector htCPUs; + os.getAvailableCPUs(physicalCPUs, htCPUs); + printCPUs("Available physical CPUs", physicalCPUs); + printCPUs("Available HT CPUs", htCPUs); + + } + catch (const except::Throwable& t) + { + std::cerr << "Caught throwable: " << t.toString() << std::endl; + return 1; + } + catch (...) + { + std::cerr << "Caught unnamed exception" << std::endl; + return 1; + } + return 0; +} diff --git a/externals/coda-oss/modules/c++/sys/unittests/test_symlink.cpp b/externals/coda-oss/modules/c++/sys/tests/test_symlink.cpp similarity index 74% rename from externals/coda-oss/modules/c++/sys/unittests/test_symlink.cpp rename to externals/coda-oss/modules/c++/sys/tests/test_symlink.cpp index 37aa42c2a..cfd30c5d7 100644 --- a/externals/coda-oss/modules/c++/sys/unittests/test_symlink.cpp +++ b/externals/coda-oss/modules/c++/sys/tests/test_symlink.cpp @@ -1,93 +1,78 @@ -/* ========================================================================= - * This file is part of sys-c++ - * ========================================================================= - * - * (C) Copyright 2004 - 2014, MDA Information Systems LLC - * - * sys-c++ is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation; either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this program; If not, - * see . - * - */ - -#include -//#include -#include -#include -#include - -#include "TestCase.h" - -namespace -{ - -TEST_CASE(testCreateSymlink) -{ - try - { - sys::OS os; - sys::Path p("this/that"); - p.makeDirectory(true); - os.createSymlink(p.getAbsolutePath(), "symlink"); - - TEST_ASSERT(os.exists("symlink")); - TEST_ASSERT(os.exists("this/that")); - - std::string output = "Test message! Testing 1, 2, 3; testing 1, 2, 3?"; - - std::ofstream fout; - fout.open("this/that/test.txt"); - fout << output; - fout.close(); - - std::ifstream fin; - fin.open("symlink/test.txt"); - std::string input; - std::getline(fin, input); - fin.close(); - - TEST_ASSERT_EQ(input, output); - - os.remove("this"); - os.removeSymlink("symlink"); - - TEST_ASSERT(!os.exists("symlink")); - TEST_ASSERT(!os.exists("this/that")); - } - catch (const std::exception& ex) - { - std::cerr << "Caught std::exception: " << ex.what() << std::endl; - TEST_ASSERT(false); - } - catch (const except::Exception& ex) - { - std::cerr << "Caught except::exception: " << ex.getMessage() - << std::endl; - TEST_ASSERT(false); - } - catch (...) - { - std::cerr << "Caught unknown exception\n"; - TEST_ASSERT(false); - } -} - -} - - -int main(int, char**) -{ - TEST_CHECK(testCreateSymlink); - - return 0; -} +/* ========================================================================= + * This file is part of sys-c++ + * ========================================================================= + * + * (C) Copyright 2004 - 2014, MDA Information Systems LLC + * + * sys-c++ is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this program; If not, + * see . + * + */ + +#include +#include +#include +#include +#include + +int main(int /*argc*/, char** /*argv*/) +{ + try + { + sys::OS os; + sys::Path p("this/that"); + p.makeDirectory(true); + os.createSymlink(p.getAbsolutePath(), "symlink"); + + assert(os.exists("symlink")); + assert(os.exists("this/that")); + + std::string output = "Test message! Testing 1, 2, 3; testing 1, 2, 3?"; + + std::ofstream fout; + fout.open("this/that/test.txt"); + fout << output; + fout.close(); + + std::ifstream fin; + fin.open("symlink/test.txt"); + std::string input; + std::getline(fin, input); + fin.close(); + + assert(input == output); + + os.remove("this"); + os.removeSymlink("symlink"); + + assert(!os.exists("symlink")); + assert(!os.exists("this/that")); + + return 0; + } + catch (const std::exception& ex) + { + std::cerr << "Caught std::exception: " << ex.what() << std::endl; + } + catch (const except::Exception& ex) + { + std::cerr << "Caught except::exception: " << ex.getMessage() + << std::endl; + } + catch (...) + { + std::cerr << "Caught unknown exception\n"; + } + return 1; +} diff --git a/externals/coda-oss/modules/c++/sys/wscript b/externals/coda-oss/modules/c++/sys/wscript index fd51787f8..f336182b1 100644 --- a/externals/coda-oss/modules/c++/sys/wscript +++ b/externals/coda-oss/modules/c++/sys/wscript @@ -1,79 +1,12 @@ NAME = 'sys' MAINTAINER = 'jmrandol@users.sourceforge.net' VERSION = '1.2' -MODULE_DEPS = 'str except' +MODULE_DEPS = 'str except config' USELIB = 'THREAD DL RT SOCKET' UNITTEST_DEPS = 'mt' TEST_FILTER = 'MMapReadOnlyTest.cpp ProcessTest1.cpp' -options = distclean = lambda p: None - -from build import writeConfig -from waflib import Utils - -def configure(conf): - - # this check defines HAVE_CLOCK_GETTIME, undefine it to keep it out of dumpenv - if conf.check_cc(lib='rt', uselib_store='RT', function_name='clock_gettime', header_name='time.h', mandatory=False): - conf.undefine('HAVE_CLOCK_GETTIME') - - # callback function to check for all #defines used by the sys module - def sys_callback(conf): - conf.check_cc(header_name='pthread.h', mandatory=False) - conf.check_cc(header_name='execinfo.h', mandatory=False) - conf.check_cc(function_name='clock_gettime', header_name='time.h', mandatory=False) - conf.check_cc(header_name="atomic.h", mandatory=False) - conf.check_cc(header_name="sys/time.h", mandatory=False) - conf.check_cc(function_name='localtime_r', header_name="time.h", mandatory=False) - conf.check_cc(function_name='gmtime_r', header_name="time.h", mandatory=False) - conf.check_cc(function_name='setenv', header_name="stdlib.h", mandatory=False) - conf.check_cc(function_name='posix_memalign', header_name="stdlib.h", mandatory=False) - conf.check_cc(function_name='memalign', header_name="stdlib.h", mandatory=False) - types_str = ''' - #include - int isBigEndian() - { - long one = 1; - return !(*((char *)(&one))); - } - int main() - { - if (isBigEndian()) printf("bigendian=True\\n"); - else printf("bigendian=False\\n"); - printf("sizeof_size_t=%d\\n", sizeof(size_t)); - return 0; - } - ''' - - # Visual Studio 2013 has nullptr but not constexpr. Need to check for - # both in here to make sure we have full C++11 support... otherwise, - # long-term we may need multiple separate configure checks and - # corresponding defines - cpp11_str = ''' - int main() - { - constexpr void* FOO = nullptr; - } - ''' - #find out the size of some types, etc. - # TODO: This is not using the 32 vs. 64 bit linker flags, so if you're - # building with --enable-32bit on 64 bit Linux, sizeof(size_t) will - # erroneously be 8 here. - output = conf.check(fragment=types_str, execute=1, msg='Checking system type sizes', define_ret=True) - t = Utils.str_to_dict(output or '') - for k, v in t.items(): - try: - v = int(v) - except: - v = v.strip() - if v == 'True': - v = True - elif v == 'False': - v = False - conf.define(k.upper(), v) - conf.check_cxx(fragment = cpp11_str, execute=1, msg='Checking for C++11 support', define_name='__CODA_CPP11', mandatory=False) - writeConfig(conf, sys_callback, NAME) +options = configure = distclean = lambda p: None def build(bld): bld.module(**globals()) - diff --git a/externals/coda-oss/modules/c++/types/include/types/Range.h b/externals/coda-oss/modules/c++/types/include/types/Range.h index e93ec64ed..2ba983ec8 100644 --- a/externals/coda-oss/modules/c++/types/include/types/Range.h +++ b/externals/coda-oss/modules/c++/types/include/types/Range.h @@ -123,6 +123,28 @@ struct Range } } + /*! + * Determine if a given range touches our range. Touching ranges do not + * overlap, but can be placed next to one another (irrespective of order) + * with no missing values in between. + * + * If either of the ranges is empty, touching is defined as always false. + * + * \param rhs Range to compare with + * + * \return True if the ranges touch, false otherwise + */ + bool touches(const types::Range& rhs) const + { + if (empty() || rhs.empty()) + { + return false; + } + + return (mStartElement == rhs.endElement()) || + (rhs.mStartElement == endElement()); + } + /*! * \param startElementToTest The start element * \param numElementsToTest The total number of elements to check diff --git a/externals/coda-oss/modules/c++/types/unittests/test_range.cpp b/externals/coda-oss/modules/c++/types/unittests/test_range.cpp index 43c3d316a..ef9c24c9b 100644 --- a/externals/coda-oss/modules/c++/types/unittests/test_range.cpp +++ b/externals/coda-oss/modules/c++/types/unittests/test_range.cpp @@ -53,10 +53,54 @@ TEST_CASE(TestGetNumSharedElements) // Ranges are the same - should share [100, 150) TEST_ASSERT_EQ(range.getNumSharedElements(100, 50), 50); } + +TEST_CASE(TestTouches) +{ + // Ranges do not overlap or touch -- touches(...) returns false + { + const types::Range A(5, 4); // [5, 8] + const types::Range B(12, 1); // [12, 12] + TEST_ASSERT_FALSE(A.touches(B)); + TEST_ASSERT_FALSE(B.touches(A)); + } + + // Ranges overlap -- touches(...) returns false + { + const types::Range A(5, 4); // [5, 8] + const types::Range B(8, 1); // [8, 8] + TEST_ASSERT_FALSE(A.touches(B)); + TEST_ASSERT_FALSE(B.touches(A)); + } + + // Ranges touch -- touches(...) returns true + { + const types::Range A(5, 4); // [5, 8] + const types::Range B(9, 1); // [9, 9] + TEST_ASSERT_TRUE(A.touches(B)); + TEST_ASSERT_TRUE(B.touches(A)); + } + + // One of the ranges is empty -- touches(...) returns false + { + const types::Range A(10, 0); // [10, 0) + const types::Range B(10, 10); // [10, 20) + TEST_ASSERT_FALSE(A.touches(B)); + TEST_ASSERT_FALSE(B.touches(A)); + } + + // Both of the ranges are empty -- touches(...) returns false + { + const types::Range A(10, 0); // [10, 0) + const types::Range B(10, 0); // [10, 20) + TEST_ASSERT_FALSE(A.touches(B)); + TEST_ASSERT_FALSE(B.touches(A)); + } +} } int main(int /*argc*/, char** /*argv*/) { TEST_CHECK(TestGetNumSharedElements); + TEST_CHECK(TestTouches); return 0; } diff --git a/externals/coda-oss/modules/drivers/jpeg/wscript b/externals/coda-oss/modules/drivers/jpeg/wscript index aad6d738e..cb2c3304b 100644 --- a/externals/coda-oss/modules/drivers/jpeg/wscript +++ b/externals/coda-oss/modules/drivers/jpeg/wscript @@ -36,13 +36,30 @@ def options(opt): from build import writeConfig -def configure(conf): + +def toggleSharedBuild(conf, driverNode): + decoration = conf.env['declspec_decoration'] + + jmorecfg = os.path.join(driverNode.relpath(), 'jmorecfg.h') + with open(jmorecfg, 'r') as f: + contents = f.readlines() + with open(jmorecfg, 'w') as f: + for line in contents: + if line.startswith('#define EXTERN(type)'): + f.write('#define EXTERN(type) extern {0} type\n'.format(decoration)) + elif line.startswith('#define GLOBAL(type)'): + f.write('#define GLOBAL(type) {0} type\n'.format(decoration)) + else: + f.write(line) + + +def configure(conf): if Options.options.enable_jpeg: jpegHome = Options.options.jpeg_home if jpegHome: - if conf.check(lib='jpeg' if sys.platform != 'win32' else 'libjpeg', - uselib_store='JPEG', + if conf.check(lib='jpeg' if sys.platform != 'win32' else 'libjpeg', + uselib_store='JPEG', header_name='jpeglib.h', fragment=frag, libpath=join(jpegHome, 'lib'), includes=join(jpegHome, 'include'), @@ -54,34 +71,36 @@ def configure(conf): # Build is off by default when tarfile not found, fail if forced on if not os.path.exists(join(conf.path.abspath(), tarfile)): conf.fatal('Missing libjpeg tarfile') - + # callback function to check for all #defines used by the jpeg driver def jpeg_callback(conf): # These are needed by jmorecfg.h. We could just set these defines # rather than bothering with the conf check. conf.define('HAVE_PROTOTYPES', 1) conf.check_cc(fragment='int main(){unsigned short; return 0;}', - define_name='HAVE_UNSIGNED_SHORT', msg='Checking for unsigned short', + define_name='HAVE_UNSIGNED_SHORT', msg='Checking for unsigned short', mandatory=False) conf.check_cc(fragment='int main(){unsigned char i; return 0;}', - define_name='HAVE_UNSIGNED_CHAR', msg='Checking for unsigned char', + define_name='HAVE_UNSIGNED_CHAR', msg='Checking for unsigned char', mandatory=False) conf.check_cc(header_name="stddef.h", mandatory=False) conf.check_cc(header_name="stdlib.h", mandatory=False) conf.check_cc(header_name="locale.h", mandatory=False) - + conf.env['MAKE_JPEG'] = True conf.msg('Building local lib', 'jpeg (libjpeg)') - + untarFile(path=conf.path, fname=tarfile) - + # make jconfig.h driverNode = conf.path.make_node(sourcepath) writeConfig(conf, jpeg_callback, 'jpeg', infile='jconfig.cfg', outfile='jconfig.h', path=driverNode, feature='handleDefs') - + + toggleSharedBuild(conf, driverNode) + else: - if conf.check(lib='jpeg' if sys.platform != 'win32' else 'libjpeg', + if conf.check(lib='jpeg' if sys.platform != 'win32' else 'libjpeg', uselib_store='JPEG', header_name='jpeglib.h', fragment=frag, msg='Checking for library libjpeg', @@ -93,28 +112,28 @@ def configure(conf): def build(bld): variant = bld.env['VARIANT'] env = bld.all_envs[variant] - + driversNode = bld.path sourceFiles = [] driverNode = driversNode.make_node(sourcepath) - + if 'MAKE_JPEG' in env: - sources = ('cdjpeg.c', 'jaricom.c', 'jcapimin.c', 'jcapistd.c', - 'jcarith.c', 'jccoefct.c', 'jccolor.c', 'jcdctmgr.c', - 'jchuff.c', 'jcinit.c', 'jcmainct.c', 'jcmarker.c', - 'jcmaster.c', 'jcomapi.c', 'jcparam.c', 'jcprepct.c', - 'jcsample.c', 'jctrans.c', 'jdapimin.c', 'jdapistd.c', - 'jdarith.c', 'jdatadst.c', 'jdatasrc.c', 'jdcoefct.c', - 'jdcolor.c', 'jddctmgr.c', 'jdhuff.c', 'jdinput.c', - 'jdmainct.c', 'jdmarker.c', 'jdmaster.c', 'jdmerge.c', - 'jdpostct.c', 'jdsample.c', 'jdtrans.c', 'jerror.c', - 'jfdctflt.c', 'jfdctfst.c', 'jfdctint.c', 'jidctflt.c', - 'jidctfst.c', 'jidctint.c', 'jmemmgr.c', 'jmemnobs.c', - 'jquant1.c', 'jquant2.c', 'jutils.c', 'rdbmp.c', - 'rdcolmap.c', 'rdgif.c', 'rdppm.c', 'rdrle.c', - 'rdswitch.c', 'rdtarga.c', 'transupp.c', 'wrbmp.c', + sources = ('cdjpeg.c', 'jaricom.c', 'jcapimin.c', 'jcapistd.c', + 'jcarith.c', 'jccoefct.c', 'jccolor.c', 'jcdctmgr.c', + 'jchuff.c', 'jcinit.c', 'jcmainct.c', 'jcmarker.c', + 'jcmaster.c', 'jcomapi.c', 'jcparam.c', 'jcprepct.c', + 'jcsample.c', 'jctrans.c', 'jdapimin.c', 'jdapistd.c', + 'jdarith.c', 'jdatadst.c', 'jdatasrc.c', 'jdcoefct.c', + 'jdcolor.c', 'jddctmgr.c', 'jdhuff.c', 'jdinput.c', + 'jdmainct.c', 'jdmarker.c', 'jdmaster.c', 'jdmerge.c', + 'jdpostct.c', 'jdsample.c', 'jdtrans.c', 'jerror.c', + 'jfdctflt.c', 'jfdctfst.c', 'jfdctint.c', 'jidctflt.c', + 'jidctfst.c', 'jidctint.c', 'jmemmgr.c', 'jmemnobs.c', + 'jquant1.c', 'jquant2.c', 'jutils.c', 'rdbmp.c', + 'rdcolmap.c', 'rdgif.c', 'rdppm.c', 'rdrle.c', + 'rdswitch.c', 'rdtarga.c', 'transupp.c', 'wrbmp.c', 'wrgif.c', 'wrppm.c', 'wrrle.c', 'wrtarga.c') - + jpeglib = bld(features='c c%s add_targets' % env['LIB_TYPE'] or 'stlib', includes=['.'], export_includes=['.'], @@ -126,16 +145,16 @@ def build(bld): env=env.derive(), name='JPEG', targets_to_add=[]) - + if env['install_libs']: jpeglib.install_path = env['install_libdir'] - + if env['install_headers']: bld(features='install_tgt', install_path=env['install_includedir'], - dir=driverNode, pattern=('jpeglib.h', 'jconfig.h', 'jmorecfg.h'), + dir=driverNode, pattern=('jpeglib.h', 'jconfig.h', 'jmorecfg.h'), name='JPEG_HEADERS_INSTALL') jpeglib.targets_to_add += ['JPEG_HEADERS_INSTALL'] - + if env['install_source']: sourceFiles.append(tarfile) jpeglib.targets_to_add += ['JPEG_SOURCE_INSTALL'] @@ -150,7 +169,7 @@ def build(bld): def distclean(context): # remove the untarred directories import shutil - dirs = filter(lambda x: exists(join(context.path.abspath(), x)), + dirs = filter(lambda x: exists(join(context.path.abspath(), x)), [sourcepath]) for d in dirs: try: diff --git a/externals/coda-oss/modules/drivers/xml/xerces/wscript b/externals/coda-oss/modules/drivers/xml/xerces/wscript index f79a7c2c1..d2e45dfb7 100644 --- a/externals/coda-oss/modules/drivers/xml/xerces/wscript +++ b/externals/coda-oss/modules/drivers/xml/xerces/wscript @@ -196,7 +196,7 @@ def configure(conf): conf.define('HAVE_STD_NAMESPACE', 1) conf.define('XERCES_AUTOCONF', 1) - conf.define('XERCES_PLATFORM_EXPORT', '', quote=False) + conf.define('XERCES_PLATFORM_EXPORT', conf.env['declspec_decoration'], quote=False) conf.define('XERCES_PLATFORM_IMPORT', '', quote=False) conf.define('XERCES_HAS_CPP_NAMESPACE', 1) conf.define('XERCES_STD_NAMESPACE', 1) @@ -295,7 +295,9 @@ def build(bld): # build the library libType = env['LIB_TYPE'] or 'stlib' - xercesDefines = env['DEFINES'] + ['HAVE_CONFIG_H'] + xercesDefines = env['DEFINES'] + ['HAVE_CONFIG_H', 'XERCES_BUILDING_LIBRARY'] + if libType == 'shlib': + xercesDefines.append('DLL_EXPORT') xerces = bld(features='c c%s add_targets' % env['LIB_TYPE'] or 'stlib', source=sources, includes=['.', 'src'], export_includes=['src'], diff --git a/externals/coda-oss/modules/python/io/source/generated/coda_io.py b/externals/coda-oss/modules/python/io/source/generated/coda_io.py index 6eb3939b0..f4d0c7f48 100644 --- a/externals/coda-oss/modules/python/io/source/generated/coda_io.py +++ b/externals/coda-oss/modules/python/io/source/generated/coda_io.py @@ -368,6 +368,11 @@ def str(self): """str(StringStream self) -> std::string""" return _coda_io.StringStream_str(self) + + def writeBytes(self, bytes): + """writeBytes(StringStream self, PyObject * bytes)""" + return _coda_io.StringStream_writeBytes(self, bytes) + __swig_destroy__ = _coda_io.delete_StringStream __del__ = lambda self: None StringStream_swigregister = _coda_io.StringStream_swigregister diff --git a/externals/coda-oss/modules/python/io/source/generated/coda_io_wrap.cxx b/externals/coda-oss/modules/python/io/source/generated/coda_io_wrap.cxx index d85e48af9..5f32b2553 100644 --- a/externals/coda-oss/modules/python/io/source/generated/coda_io_wrap.cxx +++ b/externals/coda-oss/modules/python/io/source/generated/coda_io_wrap.cxx @@ -3138,7 +3138,6 @@ namespace swig { #include "import/sys.h" #include "import/io.h" using namespace io; - #define SWIG_PYTHON_STRICT_BYTE_CHAR SWIGINTERNINLINE PyObject* @@ -3623,6 +3622,9 @@ SWIG_From_std_string (const std::string& s) return SWIG_FromCharPtrAndSize(s.data(), s.size()); } +SWIGINTERN void io_StringStream_writeBytes(io::StringStream *self,PyObject *bytes){ + self->write(PyBytes_AsString(bytes), PyBytes_Size(bytes)); + } SWIGINTERNINLINE PyObject* SWIG_From_bool (bool value) @@ -5276,6 +5278,30 @@ SWIGINTERN PyObject *_wrap_StringStream_str(PyObject *SWIGUNUSEDPARM(self), PyOb } +SWIGINTERN PyObject *_wrap_StringStream_writeBytes(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *resultobj = 0; + io::StringStream *arg1 = (io::StringStream *) 0 ; + PyObject *arg2 = (PyObject *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject * obj0 = 0 ; + PyObject * obj1 = 0 ; + + if (!PyArg_ParseTuple(args,(char *)"OO:StringStream_writeBytes",&obj0,&obj1)) SWIG_fail; + res1 = SWIG_ConvertPtr(obj0, &argp1,SWIGTYPE_p_io__StringStream, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "StringStream_writeBytes" "', argument " "1"" of type '" "io::StringStream *""'"); + } + arg1 = reinterpret_cast< io::StringStream * >(argp1); + arg2 = obj1; + io_StringStream_writeBytes(arg1,arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + SWIGINTERN PyObject *_wrap_delete_StringStream(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { PyObject *resultobj = 0; io::StringStream *arg1 = (io::StringStream *) 0 ; @@ -6997,6 +7023,7 @@ static PyMethodDef SwigMethods[] = { ""}, { (char *)"StringStream_reset", _wrap_StringStream_reset, METH_VARARGS, (char *)"StringStream_reset(StringStream self)"}, { (char *)"StringStream_str", _wrap_StringStream_str, METH_VARARGS, (char *)"StringStream_str(StringStream self) -> std::string"}, + { (char *)"StringStream_writeBytes", _wrap_StringStream_writeBytes, METH_VARARGS, (char *)"StringStream_writeBytes(StringStream self, PyObject * bytes)"}, { (char *)"delete_StringStream", _wrap_delete_StringStream, METH_VARARGS, (char *)"delete_StringStream(StringStream self)"}, { (char *)"StringStream_swigregister", StringStream_swigregister, METH_VARARGS, NULL}, { (char *)"new_NullInputStream", _wrap_new_NullInputStream, METH_VARARGS, (char *)"new_NullInputStream(sys::SSize_T size) -> NullInputStream"}, diff --git a/externals/coda-oss/modules/python/io/source/io.i b/externals/coda-oss/modules/python/io/source/io.i index bf138b3df..105af6c77 100644 --- a/externals/coda-oss/modules/python/io/source/io.i +++ b/externals/coda-oss/modules/python/io/source/io.i @@ -30,7 +30,6 @@ #include "import/sys.h" #include "import/io.h" using namespace io; - #define SWIG_PYTHON_STRICT_BYTE_CHAR %} %include "io/InputStream.h" @@ -57,3 +56,11 @@ %include "io/FileInputStreamOS.h" %include "io/FileOutputStreamOS.h" + +%extend io::StringStream +{ + void writeBytes(PyObject* bytes) + { + $self->write(PyBytes_AsString(bytes), PyBytes_Size(bytes)); + } +} diff --git a/externals/coda-oss/modules/python/io/unittests/test_string_conversions.py b/externals/coda-oss/modules/python/io/unittests/test_string_conversions.py new file mode 100644 index 000000000..7a9e99766 --- /dev/null +++ b/externals/coda-oss/modules/python/io/unittests/test_string_conversions.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python + +""" + * ========================================================================= + * This file is part of io-python + * ========================================================================= + * + * (C) Copyright 2019, MDA Information Systems LLC + * + * io-python is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this program; If not, + * see . + * + * +""" + +import unittest +import sys +from coda.coda_io import StringStream + + +class TestStringStream(unittest.TestCase): + def test_write_string(self): + stream = StringStream() + stream.write('text') + self.assertEqual(stream.str(), 'text') + + def test_write_bytes(self): + stream = StringStream() + if sys.version_info[0] == 3: + bytesInput = bytes('text', 'utf-8') + else: + # There's no reason to ever want to do this, but just to prove + # that nothing breaks... + bytesInput = bytes('text') + stream.writeBytes(bytesInput) + self.assertEqual(stream.str(), 'text') + + +if __name__ == '__main__': + unittest.main() diff --git a/externals/coda-oss/modules/python/mt/source/generated/mt.py b/externals/coda-oss/modules/python/mt/source/generated/mt.py index 11581e391..ba8f2a7cc 100644 --- a/externals/coda-oss/modules/python/mt/source/generated/mt.py +++ b/externals/coda-oss/modules/python/mt/source/generated/mt.py @@ -163,9 +163,12 @@ class ThreadGroup(_object): __getattr__ = lambda self, name: _swig_getattr(self, ThreadGroup, name) __repr__ = _swig_repr - def __init__(self): - """__init__(mt::ThreadGroup self) -> ThreadGroup""" - this = _mt.new_ThreadGroup() + def __init__(self, *args): + """ + __init__(mt::ThreadGroup self, bool pinToCPU) -> ThreadGroup + __init__(mt::ThreadGroup self) -> ThreadGroup + """ + this = _mt.new_ThreadGroup(*args) try: self.this.append(this) except __builtin__.Exception: @@ -178,6 +181,23 @@ def joinAll(self): return _mt.ThreadGroup_joinAll(self) + def isPinToCPUEnabled(self): + """isPinToCPUEnabled(ThreadGroup self) -> bool""" + return _mt.ThreadGroup_isPinToCPUEnabled(self) + + + def getDefaultPinToCPU(): + """getDefaultPinToCPU() -> bool""" + return _mt.ThreadGroup_getDefaultPinToCPU() + + getDefaultPinToCPU = staticmethod(getDefaultPinToCPU) + + def setDefaultPinToCPU(newDefault): + """setDefaultPinToCPU(bool newDefault)""" + return _mt.ThreadGroup_setDefaultPinToCPU(newDefault) + + setDefaultPinToCPU = staticmethod(setDefaultPinToCPU) + def createThread(self, runnable): """createThread(ThreadGroup self, PyObject * runnable)""" return _mt.ThreadGroup_createThread(self, runnable) @@ -185,6 +205,14 @@ def createThread(self, runnable): ThreadGroup_swigregister = _mt.ThreadGroup_swigregister ThreadGroup_swigregister(ThreadGroup) +def ThreadGroup_getDefaultPinToCPU(): + """ThreadGroup_getDefaultPinToCPU() -> bool""" + return _mt.ThreadGroup_getDefaultPinToCPU() + +def ThreadGroup_setDefaultPinToCPU(newDefault): + """ThreadGroup_setDefaultPinToCPU(bool newDefault)""" + return _mt.ThreadGroup_setDefaultPinToCPU(newDefault) + # This file is compatible with both classic and new-style classes. diff --git a/externals/coda-oss/modules/python/mt/source/generated/mt_wrap.cxx b/externals/coda-oss/modules/python/mt/source/generated/mt_wrap.cxx index 745b45b60..9c371c32a 100644 --- a/externals/coda-oss/modules/python/mt/source/generated/mt_wrap.cxx +++ b/externals/coda-oss/modules/python/mt/source/generated/mt_wrap.cxx @@ -3385,6 +3385,70 @@ SWIGINTERN PyObject *mt_ThreadPlanner_getThreadInfo(mt::ThreadPlanner *self,size return Py_BuildValue("Nnn", PyBool_FromLong(isValid), startElement, numElementsThisThread); } + +SWIGINTERN int +SWIG_AsVal_long (PyObject *obj, long* val) +{ +#if PY_VERSION_HEX < 0x03000000 + if (PyInt_Check(obj)) { + if (val) *val = PyInt_AsLong(obj); + return SWIG_OK; + } else +#endif + if (PyLong_Check(obj)) { + long v = PyLong_AsLong(obj); + if (!PyErr_Occurred()) { + if (val) *val = v; + return SWIG_OK; + } else { + PyErr_Clear(); + return SWIG_OverflowError; + } + } +#ifdef SWIG_PYTHON_CAST_MODE + { + int dispatch = 0; + long v = PyInt_AsLong(obj); + if (!PyErr_Occurred()) { + if (val) *val = v; + return SWIG_AddCast(SWIG_OK); + } else { + PyErr_Clear(); + } + if (!dispatch) { + double d; + int res = SWIG_AddCast(SWIG_AsVal_double (obj,&d)); + if (SWIG_IsOK(res) && SWIG_CanCastAsInteger(&d, LONG_MIN, LONG_MAX)) { + if (val) *val = (long)(d); + return res; + } + } + } +#endif + return SWIG_TypeError; +} + + +SWIGINTERN int +SWIG_AsVal_bool (PyObject *obj, bool *val) +{ + int r; + if (!PyBool_Check(obj)) + return SWIG_ERROR; + r = PyObject_IsTrue(obj); + if (r == -1) + return SWIG_ERROR; + if (val) *val = r ? true : false; + return SWIG_OK; +} + + +SWIGINTERNINLINE PyObject* + SWIG_From_bool (bool value) +{ + return PyBool_FromLong(value ? 1 : 0); +} + SWIGINTERN void mt_ThreadGroup_createThread(mt::ThreadGroup *self,PyObject *runnable){ sys::Runnable *n; if (SWIG_ConvertPtr(runnable, (void **) &n, SWIGTYPE_p_sys__Runnable, SWIG_POINTER_DISOWN) == -1) @@ -3791,7 +3855,59 @@ SWIGINTERN PyObject *ThreadPlanner_swigregister(PyObject *SWIGUNUSEDPARM(self), return SWIG_Py_Void(); } -SWIGINTERN PyObject *_wrap_new_ThreadGroup(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { +SWIGINTERN PyObject *_wrap_new_ThreadGroup__SWIG_0(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *resultobj = 0; + bool arg1 ; + bool val1 ; + int ecode1 = 0 ; + PyObject * obj0 = 0 ; + mt::ThreadGroup *result = 0 ; + + if (!PyArg_ParseTuple(args,(char *)"O:new_ThreadGroup",&obj0)) SWIG_fail; + ecode1 = SWIG_AsVal_bool(obj0, &val1); + if (!SWIG_IsOK(ecode1)) { + SWIG_exception_fail(SWIG_ArgError(ecode1), "in method '" "new_ThreadGroup" "', argument " "1"" of type '" "bool""'"); + } + arg1 = static_cast< bool >(val1); + { + try + { + result = (mt::ThreadGroup *)new mt::ThreadGroup(arg1); + } + catch (const std::exception& e) + { + if (!PyErr_Occurred()) + { + PyErr_SetString(PyExc_RuntimeError, e.what()); + } + } + catch (const except::Exception& e) + { + if (!PyErr_Occurred()) + { + PyErr_SetString(PyExc_RuntimeError, e.getMessage().c_str()); + } + } + catch (...) + { + if (!PyErr_Occurred()) + { + PyErr_SetString(PyExc_RuntimeError, "Unknown error"); + } + } + if (PyErr_Occurred()) + { + SWIG_fail; + } + } + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_mt__ThreadGroup, SWIG_POINTER_NEW | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_new_ThreadGroup__SWIG_1(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { PyObject *resultobj = 0; mt::ThreadGroup *result = 0 ; @@ -3834,6 +3950,41 @@ SWIGINTERN PyObject *_wrap_new_ThreadGroup(PyObject *SWIGUNUSEDPARM(self), PyObj } +SWIGINTERN PyObject *_wrap_new_ThreadGroup(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[2] = { + 0 + }; + Py_ssize_t ii; + + if (!PyTuple_Check(args)) SWIG_fail; + argc = args ? PyObject_Length(args) : 0; + for (ii = 0; (ii < 1) && (ii < argc); ii++) { + argv[ii] = PyTuple_GET_ITEM(args,ii); + } + if (argc == 0) { + return _wrap_new_ThreadGroup__SWIG_1(self, args); + } + if (argc == 1) { + int _v; + { + int res = SWIG_AsVal_bool(argv[0], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_new_ThreadGroup__SWIG_0(self, args); + } + } + +fail: + SWIG_SetErrorMsg(PyExc_NotImplementedError,"Wrong number or type of arguments for overloaded function 'new_ThreadGroup'.\n" + " Possible C/C++ prototypes are:\n" + " mt::ThreadGroup::ThreadGroup(bool)\n" + " mt::ThreadGroup::ThreadGroup()\n"); + return 0; +} + + SWIGINTERN PyObject *_wrap_delete_ThreadGroup(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { PyObject *resultobj = 0; mt::ThreadGroup *arg1 = (mt::ThreadGroup *) 0 ; @@ -3936,6 +4087,152 @@ SWIGINTERN PyObject *_wrap_ThreadGroup_joinAll(PyObject *SWIGUNUSEDPARM(self), P } +SWIGINTERN PyObject *_wrap_ThreadGroup_isPinToCPUEnabled(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *resultobj = 0; + mt::ThreadGroup *arg1 = (mt::ThreadGroup *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject * obj0 = 0 ; + bool result; + + if (!PyArg_ParseTuple(args,(char *)"O:ThreadGroup_isPinToCPUEnabled",&obj0)) SWIG_fail; + res1 = SWIG_ConvertPtr(obj0, &argp1,SWIGTYPE_p_mt__ThreadGroup, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ThreadGroup_isPinToCPUEnabled" "', argument " "1"" of type '" "mt::ThreadGroup const *""'"); + } + arg1 = reinterpret_cast< mt::ThreadGroup * >(argp1); + { + try + { + result = (bool)((mt::ThreadGroup const *)arg1)->isPinToCPUEnabled(); + } + catch (const std::exception& e) + { + if (!PyErr_Occurred()) + { + PyErr_SetString(PyExc_RuntimeError, e.what()); + } + } + catch (const except::Exception& e) + { + if (!PyErr_Occurred()) + { + PyErr_SetString(PyExc_RuntimeError, e.getMessage().c_str()); + } + } + catch (...) + { + if (!PyErr_Occurred()) + { + PyErr_SetString(PyExc_RuntimeError, "Unknown error"); + } + } + if (PyErr_Occurred()) + { + SWIG_fail; + } + } + resultobj = SWIG_From_bool(static_cast< bool >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ThreadGroup_getDefaultPinToCPU(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *resultobj = 0; + bool result; + + if (!PyArg_ParseTuple(args,(char *)":ThreadGroup_getDefaultPinToCPU")) SWIG_fail; + { + try + { + result = (bool)mt::ThreadGroup::getDefaultPinToCPU(); + } + catch (const std::exception& e) + { + if (!PyErr_Occurred()) + { + PyErr_SetString(PyExc_RuntimeError, e.what()); + } + } + catch (const except::Exception& e) + { + if (!PyErr_Occurred()) + { + PyErr_SetString(PyExc_RuntimeError, e.getMessage().c_str()); + } + } + catch (...) + { + if (!PyErr_Occurred()) + { + PyErr_SetString(PyExc_RuntimeError, "Unknown error"); + } + } + if (PyErr_Occurred()) + { + SWIG_fail; + } + } + resultobj = SWIG_From_bool(static_cast< bool >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ThreadGroup_setDefaultPinToCPU(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *resultobj = 0; + bool arg1 ; + bool val1 ; + int ecode1 = 0 ; + PyObject * obj0 = 0 ; + + if (!PyArg_ParseTuple(args,(char *)"O:ThreadGroup_setDefaultPinToCPU",&obj0)) SWIG_fail; + ecode1 = SWIG_AsVal_bool(obj0, &val1); + if (!SWIG_IsOK(ecode1)) { + SWIG_exception_fail(SWIG_ArgError(ecode1), "in method '" "ThreadGroup_setDefaultPinToCPU" "', argument " "1"" of type '" "bool""'"); + } + arg1 = static_cast< bool >(val1); + { + try + { + mt::ThreadGroup::setDefaultPinToCPU(arg1); + } + catch (const std::exception& e) + { + if (!PyErr_Occurred()) + { + PyErr_SetString(PyExc_RuntimeError, e.what()); + } + } + catch (const except::Exception& e) + { + if (!PyErr_Occurred()) + { + PyErr_SetString(PyExc_RuntimeError, e.getMessage().c_str()); + } + } + catch (...) + { + if (!PyErr_Occurred()) + { + PyErr_SetString(PyExc_RuntimeError, "Unknown error"); + } + } + if (PyErr_Occurred()) + { + SWIG_fail; + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + SWIGINTERN PyObject *_wrap_ThreadGroup_createThread(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { PyObject *resultobj = 0; mt::ThreadGroup *arg1 = (mt::ThreadGroup *) 0 ; @@ -4008,9 +4305,15 @@ static PyMethodDef SwigMethods[] = { { (char *)"ThreadPlanner_getThreadInfo", _wrap_ThreadPlanner_getThreadInfo, METH_VARARGS, (char *)"ThreadPlanner_getThreadInfo(ThreadPlanner self, size_t threadNum) -> PyObject *"}, { (char *)"delete_ThreadPlanner", _wrap_delete_ThreadPlanner, METH_VARARGS, (char *)"delete_ThreadPlanner(ThreadPlanner self)"}, { (char *)"ThreadPlanner_swigregister", ThreadPlanner_swigregister, METH_VARARGS, NULL}, - { (char *)"new_ThreadGroup", _wrap_new_ThreadGroup, METH_VARARGS, (char *)"new_ThreadGroup() -> ThreadGroup"}, + { (char *)"new_ThreadGroup", _wrap_new_ThreadGroup, METH_VARARGS, (char *)"\n" + "ThreadGroup(bool pinToCPU)\n" + "new_ThreadGroup() -> ThreadGroup\n" + ""}, { (char *)"delete_ThreadGroup", _wrap_delete_ThreadGroup, METH_VARARGS, (char *)"delete_ThreadGroup(ThreadGroup self)"}, { (char *)"ThreadGroup_joinAll", _wrap_ThreadGroup_joinAll, METH_VARARGS, (char *)"ThreadGroup_joinAll(ThreadGroup self)"}, + { (char *)"ThreadGroup_isPinToCPUEnabled", _wrap_ThreadGroup_isPinToCPUEnabled, METH_VARARGS, (char *)"ThreadGroup_isPinToCPUEnabled(ThreadGroup self) -> bool"}, + { (char *)"ThreadGroup_getDefaultPinToCPU", _wrap_ThreadGroup_getDefaultPinToCPU, METH_VARARGS, (char *)"ThreadGroup_getDefaultPinToCPU() -> bool"}, + { (char *)"ThreadGroup_setDefaultPinToCPU", _wrap_ThreadGroup_setDefaultPinToCPU, METH_VARARGS, (char *)"ThreadGroup_setDefaultPinToCPU(bool newDefault)"}, { (char *)"ThreadGroup_createThread", _wrap_ThreadGroup_createThread, METH_VARARGS, (char *)"ThreadGroup_createThread(ThreadGroup self, PyObject * runnable)"}, { (char *)"ThreadGroup_swigregister", ThreadGroup_swigregister, METH_VARARGS, NULL}, { NULL, NULL, 0, NULL } diff --git a/externals/coda-oss/modules/python/test_utils/source/test_utils.py b/externals/coda-oss/modules/python/test_utils/source/test_utils.py index 7ee8598a7..d41aaefab 100644 --- a/externals/coda-oss/modules/python/test_utils/source/test_utils.py +++ b/externals/coda-oss/modules/python/test_utils/source/test_utils.py @@ -49,7 +49,11 @@ def runUnitTests(projectName, installDirName='install'): for test in os.listdir(os.path.join(unitTestDir, childDir)): testPathname = os.path.join(unitTestDir, childDir, test) print(testPathname) - if call([executableName(testPathname)]) != 0: + if testPathname.endswith('.py'): + command = ['python', testPathname] + else: + command = [executableName(testPathname)] + if call(command) != 0: success = False return success diff --git a/externals/coda-oss/wscript b/externals/coda-oss/wscript index 2932fbef4..50f5030c2 100644 --- a/externals/coda-oss/wscript +++ b/externals/coda-oss/wscript @@ -1,5 +1,5 @@ import os -from build import CPPOptionsContext +from build import CPPOptionsContext, enableWafUnitTests from waflib import Scripting, Options, Context VERSION = '3.0-dev' @@ -18,11 +18,13 @@ def options(opt): opt.recurse(DIRS) def configure(conf): + conf.options.swigver = '3.0.12' conf.load(TOOLS, tooldir='./build/') conf.recurse(DIRS) def build(bld): bld.recurse(DIRS) + enableWafUnitTests(bld) def distclean(ctxt): ctxt.recurse(DIRS) diff --git a/modules/c++/nitf/source/ImageWriter.cpp b/modules/c++/nitf/source/ImageWriter.cpp index 40aa66298..a080214b2 100644 --- a/modules/c++/nitf/source/ImageWriter.cpp +++ b/modules/c++/nitf/source/ImageWriter.cpp @@ -56,7 +56,10 @@ void ImageWriter::setWriteCaching(int enable) void ImageWriter::setDirectBlockWrite(int enable) { - nitf_ImageWriter_setDirectBlockWrite(getNativeOrThrow(), enable); + if (!nitf_ImageWriter_setDirectBlockWrite(getNativeOrThrow(), enable, &error)) + { + throw nitf::NITFException(&error); + } } void ImageWriter::setPadPixel(nitf::Uint8* value, nitf::Uint32 length) diff --git a/modules/c++/nitf/tests/test_round_trip.cpp b/modules/c++/nitf/tests/test_round_trip.cpp index 9e3951515..a683901ca 100644 --- a/modules/c++/nitf/tests/test_round_trip.cpp +++ b/modules/c++/nitf/tests/test_round_trip.cpp @@ -185,7 +185,8 @@ int main(int argc, char **argv) } catch (except::Throwable & t) { - std::cout << "ERROR!: " << t.toString() << std::endl; + std::cerr << "ERROR!: " << t.toString() << std::endl; + return 1; } } diff --git a/modules/c++/nitf/unittests/test_field++.cpp b/modules/c++/nitf/unittests/test_field++.cpp index c9ebac29f..1f5069b13 100644 --- a/modules/c++/nitf/unittests/test_field++.cpp +++ b/modules/c++/nitf/unittests/test_field++.cpp @@ -20,7 +20,7 @@ * */ -#include +#include #include #include "TestCase.h" diff --git a/modules/c/cgm/wscript b/modules/c/cgm/wscript index c77f851dc..1400704c2 100644 --- a/modules/c/cgm/wscript +++ b/modules/c/cgm/wscript @@ -5,6 +5,7 @@ NAME = 'cgm' MAINTAINER = 'tzellman@users.sourceforge.net gojira_1@users.sourceforge.net' MODULE_DEPS = 'nitf nrt' LANG = 'c' +DEFINES = 'NITF_MODULE_EXPORTS' options = configure = distclean = lambda p: None diff --git a/modules/c/j2k/shared/wscript b/modules/c/j2k/shared/wscript index a54c6cd6a..d150c33a4 100644 --- a/modules/c/j2k/shared/wscript +++ b/modules/c/j2k/shared/wscript @@ -15,7 +15,7 @@ configure = options = distclean = lambda p: None def build(bld): variant = bld.env['VARIANT'] or 'default' env = bld.all_envs[variant] - + if 'HAVE_J2K' in env: pluginList = [] plugins = bld.path.ant_glob('*.c') diff --git a/modules/c/j2k/wscript b/modules/c/j2k/wscript index c16967d24..03612f257 100644 --- a/modules/c/j2k/wscript +++ b/modules/c/j2k/wscript @@ -1,7 +1,6 @@ from build import getDriverIncludes -USELIB_CHECK = 'J2K' options = configure = distclean = lambda x: None @@ -30,10 +29,11 @@ def build(bld): j2kIncludes = ['include', j2kInclude, env['OPENJPEG_INCLUDE']] # build the j2k library - lib = bld(features='c cstlib', + lib = bld(features='c c%s' % env['LIB_TYPE'] or 'stlib', includes=j2kIncludes, target='j2k-c', name='j2k-c', source=j2kSources, + defines='J2K_MODULE_EXPORTS', export_includes=j2kIncludes, env=env.derive(), path=bld.path, use='nitf-c J2K ' + j2kLayer) @@ -60,5 +60,3 @@ def build(bld): bld(features='add_targets', target='j2k-tests', targets_to_add=j2k_only_tests + j2k_nitf_tests) - - diff --git a/modules/c/nitf/include/nitf/ImageWriter.h b/modules/c/nitf/include/nitf/ImageWriter.h index 212c4f3e5..d7773e900 100644 --- a/modules/c/nitf/include/nitf/ImageWriter.h +++ b/modules/c/nitf/include/nitf/ImageWriter.h @@ -81,15 +81,19 @@ NITFAPI(int) nitf_ImageWriter_setWriteCaching * nitf_ImageWriter_setDirectBlockWrite enables/disables direct block writing. * If this is set to 1 and the number of image bands is 1, then each block of data * will be written directly to the NITF and bypass any manipulation or re-organization. - * If you know for certain that you're band sources will give you the data formatted + * If you know for certain that your band sources will give you the data formatted * precisely as required for whatever you're writing out, then enable this for better * write performance. This is most useful in conjunction with the DirectBlockSource * band source for file copies. + * + * This operation will fail if trying to enable direct block writing for multiband + * images. */ -NITFAPI(void) nitf_ImageWriter_setDirectBlockWrite +NITFAPI(NITF_BOOL) nitf_ImageWriter_setDirectBlockWrite ( nitf_ImageWriter * iWriter, /*!< Object to modify */ - int enable /*!< Enable cached writes if true */ + int enable, /*!< Enable cached writes if true */ + nitf_Error *error /*!< Error object */ ); /*! diff --git a/modules/c/nitf/source/DirectBlockSource.c b/modules/c/nitf/source/DirectBlockSource.c index 2f9515ae6..000a7e39c 100644 --- a/modules/c/nitf/source/DirectBlockSource.c +++ b/modules/c/nitf/source/DirectBlockSource.c @@ -54,14 +54,9 @@ NITFPRIV(DirectBlockSourceImpl *) toDirectBlockSource(NITF_DATA * data, } -/* - * DirectBlockSource_read - Read data function for row source - */ - -/* Instance data */ -/* Output buffer */ -NITFPRIV(NITF_BOOL) DirectBlockSource_read(NITF_DATA * data, void *buf, nitf_Off size, /* Amount to read */ - nitf_Error * error) /* For error returns */ +NITFPRIV(NITF_BOOL) DirectBlockSource_read(NITF_DATA * data, void *buf, + nitf_Off size, + nitf_Error * error) { DirectBlockSourceImpl *directBlockSource = toDirectBlockSource(data, error); const void* block; @@ -77,6 +72,15 @@ NITFPRIV(NITF_BOOL) DirectBlockSource_read(NITF_DATA * data, void *buf, nitf_Off if(!block) return NITF_FAILURE; + if (blockSize != size) + { + nitf_Error_initf(error, NITF_CTXT, + NITF_ERR_READING_FROM_FILE, + "Expected %lu bytes, but read %llu", + size, blockSize); + return NITF_FAILURE; + } + if(!directBlockSource->nextBlock(directBlockSource->algorithm, buf, block, diff --git a/modules/c/nitf/source/ImageIO.c b/modules/c/nitf/source/ImageIO.c index f587d7c3b..808576df8 100644 --- a/modules/c/nitf/source/ImageIO.c +++ b/modules/c/nitf/source/ImageIO.c @@ -5568,7 +5568,7 @@ int nitf_ImageIO_setup_P(_nitf_ImageIOControl * cntl, nitf_Error * error) * in the read case and is set to 0 in the write case. * * For reading, the first read for a given row segment reads - * all of the bands for that segment. The initial offset of 0 + * all of the requested bands for that segment. The initial offset * for the first band is the correct buffer offset for the read. * The pixel size times band offset then correctly line-up the * start buffer for the unpack operations. @@ -5582,8 +5582,12 @@ int nitf_ImageIO_setup_P(_nitf_ImageIOControl * cntl, nitf_Error * error) if (cntl->reading) { - blockIO->rwBuffer.offset.mark = bytes * band; - blockIO->rwBuffer.offset.orig = bytes * band; + /* In all cases, we are reading all bands for the block. + * If we're only requesting e.g. bands 2..n, need to adjust + * the starting point so that band 2 maps to position 0. + */ + blockIO->rwBuffer.offset.mark = bytes * (band - cntl->bandSubset[0]); + blockIO->rwBuffer.offset.orig = bytes * (band - cntl->bandSubset[0]); } else { @@ -5662,6 +5666,7 @@ nitf_ImageIOControl_construct(_nitf_ImageIO * nitf, nitf_Error * error) { _nitf_ImageIOControl *cntl; /* The result */ + nitf_Uint32 bandIdx; cntl = (_nitf_ImageIOControl *) NITF_MALLOC(sizeof(_nitf_ImageIOControl)); @@ -6082,6 +6087,18 @@ NITFPRIV(int) nitf_ImageIO_checkSubWindow(_nitf_ImageIO * nitf, } } + /* Require bandList[0] == min(bandList) */ + for (bandIdx = 1; bandIdx < subWindow->numBands; bandIdx++) + { + if (subWindow->bandList[bandIdx] < subWindow->bandList[0]) + { + nitf_Error_initf(error, NITF_CTXT, NITF_ERR_READING_FROM_FILE, + "Band <%ld> at index 0 is not the lowest band\n", + subWindow->bandList[0]); + return NITF_FAILURE; + } + } + /* Check for full image request */ *all = 0; @@ -6747,7 +6764,7 @@ NITFPRIV(int) nitf_ImageIO_readRequest(_nitf_ImageIOControl * cntl, nitf_Uint32 col; /* Block column index */ nitf_Uint32 row; /* Current row in sub-window */ nitf_Uint32 band; /* Current band in sub-window */ - _nitf_ImageIOBlock *blockIO; /* The current block IO structure */ + _nitf_ImageIOBlock *blockIO; /* The current block IO structure */ nitf = cntl->nitf; numRows = cntl->numRows; @@ -6762,17 +6779,26 @@ NITFPRIV(int) nitf_ImageIO_readRequest(_nitf_ImageIOControl * cntl, { blockIO = &(cntl->blockIO[col][band]); if (blockIO->doIO) + { if (!(*(nitf->vtbl.reader)) (blockIO, io, error)) + { return NITF_FAILURE; + } + } if (nitf->vtbl.unpack != NULL) + { (*(nitf->vtbl.unpack)) (blockIO, error); + } if (nitf->vtbl.unformat != NULL) + { (*(nitf->vtbl.unformat)) (blockIO->user.buffer + - blockIO->user.offset.mark, - blockIO->pixelCountDR, - nitf->pixel.shift); + blockIO->user.offset.mark, + blockIO->pixelCountDR, + nitf->pixel.shift); + } + /* * You have to check for last row and not call * nitf_ImageIO_nextRow because if the last row is the @@ -6782,17 +6808,24 @@ NITFPRIV(int) nitf_ImageIO_readRequest(_nitf_ImageIOControl * cntl, * the non-existant next block. */ if (row != numRows - 1) + { nitf_ImageIO_nextRow(blockIO, 0); + } if (blockIO->rowsUntil == 0) + { /* * See documentation of nitf_ImageIOBlock for an * explaination of the - 1 */ blockIO->rowsUntil = nitf->numRowsPerBlock - 1; + } else + { blockIO->rowsUntil -= 1; + } } + } } @@ -7102,19 +7135,17 @@ NITFPRIV(int) nitf_ImageIO_readFromFile(nitf_IOInterface* io, size_t count, nitf_Error * error) { - size_t bytes; /* Amount of current read */ char *bufp; /* pointer into the buffer */ - /* Seek to the offset */ - bytes = count; bufp = (char *) buffer; + /* Seek to the offset */ if (!NITF_IO_SUCCESS(nitf_IOInterface_seek(io, (nitf_Off) fileOffset, NITF_SEEK_SET, error))) { return NITF_FAILURE; } - if (!nitf_IOInterface_read(io, bufp, bytes, error)) + if (!nitf_IOInterface_read(io, bufp, count, error)) { return NITF_FAILURE; } @@ -8510,12 +8541,16 @@ void nitf_ImageIO_unpack_P_1(_nitf_ImageIOBlock * blockIO, size_t count; /* Number of pixels to transfer */ nitf_Uint32 skip; /* Source buffer skip count */ size_t i; + const nitf_Uint32 bytes = blockIO->cntl->nitf->pixel.bytes; + const nitf_Uint32 firstBand = blockIO->cntl->bandSubset[0]; + const nitf_Uint32 bandOffset = firstBand * bytes; /* Silence compiler warnings about unused variables */ (void)error; src = (nitf_Uint8 *) (blockIO->rwBuffer.buffer - + blockIO->rwBuffer.offset.mark); + + blockIO->rwBuffer.offset.mark + + bandOffset); dst = (nitf_Uint8 *) (blockIO->unpacked.buffer + blockIO->unpacked.offset.mark); count = blockIO->pixelCountFR; @@ -8538,12 +8573,16 @@ void nitf_ImageIO_unpack_P_2(_nitf_ImageIOBlock * blockIO, size_t count; /* Number of pixels to transfer */ nitf_Uint32 skip; /* Source buffer skip count */ size_t i; + const nitf_Uint32 bytes = blockIO->cntl->nitf->pixel.bytes; + const nitf_Uint32 firstBand = blockIO->cntl->bandSubset[0]; + const nitf_Uint32 bandOffset = firstBand * bytes; /* Silence compiler warnings about unused variables */ (void)error; src = (nitf_Uint16 *) (blockIO->rwBuffer.buffer - + blockIO->rwBuffer.offset.mark); + + blockIO->rwBuffer.offset.mark + + bandOffset); dst = (nitf_Uint16 *) (blockIO->unpacked.buffer + blockIO->unpacked.offset.mark); count = blockIO->pixelCountFR; @@ -8557,7 +8596,6 @@ void nitf_ImageIO_unpack_P_2(_nitf_ImageIOBlock * blockIO, return; } - void nitf_ImageIO_unpack_P_4(_nitf_ImageIOBlock * blockIO, nitf_Error * error) { @@ -8566,12 +8604,16 @@ void nitf_ImageIO_unpack_P_4(_nitf_ImageIOBlock * blockIO, size_t count; /* Number of pixels to transfer */ nitf_Uint32 skip; /* Source buffer skip count */ size_t i; + const nitf_Uint32 bytes = blockIO->cntl->nitf->pixel.bytes; + const nitf_Uint32 firstBand = blockIO->cntl->bandSubset[0]; + const nitf_Uint32 bandOffset = firstBand * bytes; /* Silence compiler warnings about unused variables */ (void)error; src = (nitf_Uint32 *) (blockIO->rwBuffer.buffer - + blockIO->rwBuffer.offset.mark); + + blockIO->rwBuffer.offset.mark + + bandOffset); dst = (nitf_Uint32 *) (blockIO->unpacked.buffer + blockIO->unpacked.offset.mark); count = blockIO->pixelCountFR; @@ -8594,12 +8636,16 @@ void nitf_ImageIO_unpack_P_8(_nitf_ImageIOBlock * blockIO, size_t count; /* Number of pixels to transfer */ nitf_Uint32 skip; /* Source buffer skip count */ size_t i; + const nitf_Uint32 bytes = blockIO->cntl->nitf->pixel.bytes; + const nitf_Uint32 firstBand = blockIO->cntl->bandSubset[0]; + const nitf_Uint32 bandOffset = firstBand * bytes; /* Silence compiler warnings about unused variables */ (void)error; src = (nitf_Uint64 *) (blockIO->rwBuffer.buffer - + blockIO->rwBuffer.offset.mark); + + blockIO->rwBuffer.offset.mark + + bandOffset); dst = (nitf_Uint64 *) (blockIO->unpacked.buffer + blockIO->unpacked.offset.mark); count = blockIO->pixelCountFR; @@ -8624,12 +8670,16 @@ void nitf_ImageIO_unpack_P_16(_nitf_ImageIOBlock * blockIO, size_t count; /* Number of pixels to transfer */ nitf_Uint32 skip; /* Source buffer skip count */ size_t i; + const nitf_Uint32 bytes = blockIO->cntl->nitf->pixel.bytes; + const nitf_Uint32 firstBand = blockIO->cntl->bandSubset[0]; + const nitf_Uint32 bandOffset = firstBand * bytes; /* Silence compiler warnings about unused variables */ (void)error; src1 = (nitf_Uint64 *) (blockIO->rwBuffer.buffer - + blockIO->rwBuffer.offset.mark); + + blockIO->rwBuffer.offset.mark + + bandOffset); dst1 = (nitf_Uint64 *) (blockIO->unpacked.buffer + blockIO->unpacked.offset.mark); src2 = src1 + 1; diff --git a/modules/c/nitf/source/ImageWriter.c b/modules/c/nitf/source/ImageWriter.c index 654de25da..bd7cac875 100644 --- a/modules/c/nitf/source/ImageWriter.c +++ b/modules/c/nitf/source/ImageWriter.c @@ -303,11 +303,24 @@ NITFAPI(int) nitf_ImageWriter_setWriteCaching(nitf_ImageWriter *imageWriter, return(nitf_ImageIO_setWriteCaching(impl->imageBlocker, enable)); } -NITFAPI(void) nitf_ImageWriter_setDirectBlockWrite(nitf_ImageWriter *imageWriter, - int enable) +NITFAPI(NITF_BOOL) nitf_ImageWriter_setDirectBlockWrite(nitf_ImageWriter *imageWriter, + int enable, + nitf_Error *error) { - ImageWriterImpl *impl = (ImageWriterImpl*)imageWriter->data; + ImageWriterImpl *impl; + nitf_Uint32 numImageBands; + + impl = (ImageWriterImpl*)imageWriter->data; + numImageBands = impl->numImageBands + impl->numMultispectralImageBands; + + if (numImageBands > 1 && enable) + { + nitf_Error_init(error, "Direct block write not supported for multi-band images", + NITF_CTXT, NITF_ERR_INVALID_PARAMETER); + return NITF_FAILURE; + } impl->directBlockWrite = enable; + return NITF_SUCCESS; } NITFAPI(NITF_BOOL) nitf_ImageWriter_setPadPixel(nitf_ImageWriter* imageWriter, diff --git a/modules/c/nitf/unittests/test_image_io.c b/modules/c/nitf/unittests/test_image_io.c new file mode 100644 index 000000000..439b0aad8 --- /dev/null +++ b/modules/c/nitf/unittests/test_image_io.c @@ -0,0 +1,923 @@ +/* ========================================================================= + * This file is part of NITRO + * ========================================================================= + * + * (C) Copyright 2004 - 2019, MDA Information Systems LLC + * + * NITRO is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this program; if not, If not, + * see . + * + */ + +#include "Test.h" +#include + +typedef struct TestSpec +{ + // Image spec + const char* imageMode; + nitf_Uint32 bitsPerPixel; + char* pixels; + nitf_Uint64 imageSize; + nitf_Uint32 numBands; + + // Read request + nitf_Uint32 startRow; + nitf_Uint32 numRows; + nitf_Uint32 startCol; + nitf_Uint32 numCols; + + const char* expectedRead; +} TestSpec; + +// Hold onto any dynamically allocated objects for a test, +// so we can pull all the initialization up into one function, +// but still hold onto everything to free it +typedef struct TestState +{ + nitf_ImageSubheader* subheader; + nitf_ImageIO* imageIO; + nitf_IOInterface* interface; + nitf_SubWindow* subwindow; + nitf_Uint32 numBands; + nitf_Uint32* bandList; + nitf_BandInfo** band; +} TestState; + +#define NUM_ROWS 16 +#define NUM_COLS 16 +#define ROWS_PER_BLOCK 4 +#define COLS_PER_BLOCK 4 + +TestState* constructTestSubheader(TestSpec* spec) +{ + nitf_Error error; + + TestState* state = (TestState*)malloc(sizeof(TestState)); + assert(state); + + nitf_ImageSubheader *subheader = nitf_ImageSubheader_construct(&error); + assert(subheader); + + nitf_ImageSubheader_setBlocking(subheader, + NUM_ROWS, NUM_COLS, + ROWS_PER_BLOCK, COLS_PER_BLOCK, + spec->imageMode, + &error); + + state->band = malloc(sizeof(nitf_BandInfo) * spec->numBands); + assert(state->band); + + nitf_Uint32 band; + for (band = 0; band < spec->numBands; ++band) + { + state->band[band] = nitf_BandInfo_construct(&error); + assert(state->band[band]); + + if (!nitf_BandInfo_init(state->band[band], + "M", " ", "N", " ", 0, 0, NULL, &error)) + { + exit(1); + } + } + + nitf_ImageSubheader_setPixelInformation( + subheader, + "INT", + spec->bitsPerPixel, spec->bitsPerPixel, + "R", + "MONO", + "VIS", + spec->numBands, + state->band, + &error); + nitf_ImageSubheader_setCompression(subheader, "NC", "", &error); + + + nitf_ImageIO* imageIO = nitf_ImageIO_construct( + subheader, 0, spec->imageSize, NULL, NULL, NULL, &error); + assert(imageIO); + + nitf_IOInterface *io = nitf_BufferAdapter_construct( + spec->pixels, spec->imageSize, 0, &error); + assert(io); + + nitf_SubWindow *subwindow = nitf_SubWindow_construct(&error); + assert(subwindow); + subwindow->startRow = spec->startRow; + subwindow->numRows = spec->numRows; + subwindow->startCol = spec->startCol; + subwindow->numCols = spec->numCols; + + state->subheader = subheader; + state->imageIO = imageIO; + state->interface = io; + state->subwindow = subwindow; + state->numBands = spec->numBands; + state->bandList = malloc(state->numBands * sizeof(nitf_Uint32)); + for (band = 0; band < state->numBands; ++band) + { + state->bandList[band] = band; + } + assert(state->bandList); + subwindow->numBands = state->numBands; + subwindow->bandList = state->bandList; + return state; +} + +void freeTestState(TestState* state) +{ + nitf_SubWindow_destruct(&state->subwindow); + nitf_IOInterface_destruct(&state->interface); + nitf_ImageIO_destruct(&state->imageIO); + nitf_ImageSubheader_destruct(&state->subheader); + free(state->bandList); + free(state); +} + +void freeBands(nitf_Uint8** bands, size_t numBands) +{ + size_t bandIndex; + for (bandIndex = 0; bandIndex < numBands; ++bandIndex) + { + if (bands[bandIndex] != NULL) + { + free(bands[bandIndex]); + } + } + free(bands); +} + +static nitf_Uint8** allocateBands(size_t numBands, size_t bytesPerBand) +{ + size_t bandIndex; + int success = 1; + + nitf_Uint8** bands = (nitf_Uint8**)calloc(numBands, sizeof(nitf_Uint8 *)); + if (bands == NULL) + { + return NULL; + } + + for (bandIndex = 0; bandIndex < numBands; ++bandIndex) + { + bands[bandIndex] = (nitf_Uint8*)calloc(sizeof(nitf_Uint8), bytesPerBand + 1); + if (bands[bandIndex] == NULL) + { + success = 0; + } + } + + /* Free memory if we failed */ + if (success == 0) + { + freeBands(bands, numBands); + bands = NULL; + } + + return bands; +} + +static NITF_BOOL doReadTest(TestSpec* spec, TestState* test) +{ + NITF_BOOL result = NITF_SUCCESS; + + const nitf_Uint32 numBands = test->subwindow->numBands; + const size_t bandSize = strlen(spec->expectedRead) / numBands; + nitf_Uint8** bands = allocateBands(numBands, bandSize); + if (bands == NULL) + { + return NITF_FAILURE; + } + + nitf_Error error; + int padded; + nitf_ImageIO_read(test->imageIO, test->interface, test->subwindow, + bands, &padded, &error); + + char* joinedBands = NULL; + joinedBands = malloc(strlen(spec->expectedRead) + 1); + if (joinedBands) + { + strcpy(joinedBands, (const char*) bands[0]); + nitf_Uint32 bandIdx; + for (bandIdx = 1; bandIdx < numBands; ++bandIdx) + { + strcat(joinedBands, (const char*) bands[bandIdx]); + } + if (strcmp((char *)joinedBands, spec->expectedRead) != 0) + { + result = NITF_FAILURE; + } + free(joinedBands); + } + + if (bands != NULL) + { + freeBands(bands, numBands); + } + + return result; +} + +static NITF_BOOL roundTripTest(TestSpec* spec, TestState* test, char* pixels) +{ + NITF_BOOL result = NITF_SUCCESS; + nitf_Error error; + const nitf_Uint32 numBands = test->subwindow->numBands; + nitf_IOInterface* writeIO = NULL; + + const size_t bandSize = strlen(spec->expectedRead) / numBands; + nitf_Uint8** bands = allocateBands(numBands, bandSize); + if (bands == NULL) + { + return NITF_FAILURE; + } + + int padded; + nitf_ImageIO_read(test->imageIO, test->interface, test->subwindow, + bands, &padded, &error); + + const size_t bufferSize = strlen(pixels); + char* buf = (char*)calloc(bufferSize + 1, sizeof(char)); + writeIO = nrt_BufferAdapter_construct(buf, bufferSize, 1, &error); + + if (writeIO != NULL) + { + nitf_ImageIO_writeSequential(test->imageIO, writeIO, &error); + nitf_ImageIO_writeRows(test->imageIO, writeIO, NUM_ROWS, bands, &error); + nitf_ImageIO_writeDone(test->imageIO, writeIO, &error); + + if (strcmp(buf, pixels) != 0) + { + result = NITF_FAILURE; + } + + nitf_IOInterface_destruct(&writeIO); + } + + if (bands != NULL) + { + freeBands(bands, numBands); + } + + return result; +} + +TEST_CASE(testPBlockOneBand) +{ + /* For image mode P, the pixels are stored one block + * at a time. This means that the first row of pixels + * (AAAABBBBCCCCDDDD) is actually the first block of + * the image. We can find the actual first image row + * by stitching together the first (numRows / numBlocksPerRow) + * blocks and taking the first row of that. + * In this case, "AAAAAAAAAAAAAAAA" + */ +#define NUM_BANDS 1 +#define pixels \ + "AAAABBBBCCCCDDDD" \ + "AAAABBBBCCCCDDDD" \ + "AAAABBBBCCCCDDDD" \ + "AAAABBBBCCCCDDDD" \ + "EEEEFFFFGGGGHHHH" \ + "EEEEFFFFGGGGHHHH" \ + "EEEEFFFFGGGGHHHH" \ + "EEEEFFFFGGGGHHHH" \ + "IIIIJJJJKKKKLLLL" \ + "IIIIJJJJKKKKLLLL" \ + "IIIIJJJJKKKKLLLL" \ + "IIIIJJJJKKKKLLLL" \ + "MMMMNNNNOOOOPPPP" \ + "MMMMNNNNOOOOPPPP" \ + "MMMMNNNNOOOOPPPP" \ + "MMMMNNNNOOOOPPPP" + + TestSpec pTypeTests[] = + { + { + "P", + 8, + pixels, + sizeof(pixels), + NUM_BANDS, + + 0, 2, + 4, 4, + + "AAAABBBB" + }, + { + "P", + 8, + pixels, + sizeof(pixels), + NUM_BANDS, + + 0, 4, + 0, NUM_COLS, + + "AAAAAAAAAAAAAAAA" + "BBBBBBBBBBBBBBBB" + "CCCCCCCCCCCCCCCC" + "DDDDDDDDDDDDDDDD" + }, + { + "P", + 8, + pixels, + sizeof(pixels), + NUM_BANDS, + + 0, NUM_ROWS, + 0, NUM_COLS, + + "AAAAAAAAAAAAAAAA" + "BBBBBBBBBBBBBBBB" + "CCCCCCCCCCCCCCCC" + "DDDDDDDDDDDDDDDD" + "EEEEEEEEEEEEEEEE" + "FFFFFFFFFFFFFFFF" + "GGGGGGGGGGGGGGGG" + "HHHHHHHHHHHHHHHH" + "IIIIIIIIIIIIIIII" + "JJJJJJJJJJJJJJJJ" + "KKKKKKKKKKKKKKKK" + "LLLLLLLLLLLLLLLL" + "MMMMMMMMMMMMMMMM" + "NNNNNNNNNNNNNNNN" + "OOOOOOOOOOOOOOOO" + "PPPPPPPPPPPPPPPP" + } + }; + +#undef NUM_BANDS +#undef pixels + + const size_t numTests = sizeof(pTypeTests) / sizeof(pTypeTests[0]); + size_t testIndex; + for (testIndex = 0; testIndex < numTests; ++testIndex) + { + TestSpec* spec = &pTypeTests[testIndex]; + TestState* test = constructTestSubheader(spec); + + TEST_ASSERT(doReadTest(spec, test)); + freeTestState(test); + } +} + +TEST_CASE(testPBlockTwoBands) +{ +#define NUM_BANDS 2 +#define pixels \ + "AaAaAaAaBbBbBbBbCcCcCcCcDdDdDdDd"\ + "AaAaAaAaBbBbBbBbCcCcCcCcDdDdDdDd"\ + "AaAaAaAaBbBbBbBbCcCcCcCcDdDdDdDd"\ + "AaAaAaAaBbBbBbBbCcCcCcCcDdDdDdDd"\ + "EeEeEeEeFfFfFfFfGgGgGgGgHhHhHhHh"\ + "EeEeEeEeFfFfFfFfGgGgGgGgHhHhHhHh"\ + "EeEeEeEeFfFfFfFfGgGgGgGgHhHhHhHh"\ + "EeEeEeEeFfFfFfFfGgGgGgGgHhHhHhHh"\ + "IiIiIiIiJjJjJjJjKkKkKkKkLlLlLlLl"\ + "IiIiIiIiJjJjJjJjKkKkKkKkLlLlLlLl"\ + "IiIiIiIiJjJjJjJjKkKkKkKkLlLlLlLl"\ + "IiIiIiIiJjJjJjJjKkKkKkKkLlLlLlLl"\ + "MmMmMmMmNnNnNnNnOoOoOoOoPpPpPpPp"\ + "MmMmMmMmNnNnNnNnOoOoOoOoPpPpPpPp"\ + "MmMmMmMmNnNnNnNnOoOoOoOoPpPpPpPp"\ + "MmMmMmMmNnNnNnNnOoOoOoOoPpPpPpPp" + + TestSpec pTypeTests[] = + { + + { + "P", + 8, + pixels, + sizeof(pixels), + NUM_BANDS, + + 0, 2, + 4, 4, + + "AAAABBBBaaaabbbb" + }, + { + "P", + 8, + pixels, + sizeof(pixels), + NUM_BANDS, + + 0, 4, + 0, NUM_COLS, + + "AAAAAAAAAAAAAAAA" + "BBBBBBBBBBBBBBBB" + "CCCCCCCCCCCCCCCC" + "DDDDDDDDDDDDDDDD" + "aaaaaaaaaaaaaaaa" + "bbbbbbbbbbbbbbbb" + "cccccccccccccccc" + "dddddddddddddddd" + }, + { + "P", + 8, + pixels, + sizeof(pixels), + NUM_BANDS, + + 0, NUM_ROWS, + 0, NUM_COLS, + + "AAAAAAAAAAAAAAAA" + "BBBBBBBBBBBBBBBB" + "CCCCCCCCCCCCCCCC" + "DDDDDDDDDDDDDDDD" + "EEEEEEEEEEEEEEEE" + "FFFFFFFFFFFFFFFF" + "GGGGGGGGGGGGGGGG" + "HHHHHHHHHHHHHHHH" + "IIIIIIIIIIIIIIII" + "JJJJJJJJJJJJJJJJ" + "KKKKKKKKKKKKKKKK" + "LLLLLLLLLLLLLLLL" + "MMMMMMMMMMMMMMMM" + "NNNNNNNNNNNNNNNN" + "OOOOOOOOOOOOOOOO" + "PPPPPPPPPPPPPPPP" + "aaaaaaaaaaaaaaaa" + "bbbbbbbbbbbbbbbb" + "cccccccccccccccc" + "dddddddddddddddd" + "eeeeeeeeeeeeeeee" + "ffffffffffffffff" + "gggggggggggggggg" + "hhhhhhhhhhhhhhhh" + "iiiiiiiiiiiiiiii" + "jjjjjjjjjjjjjjjj" + "kkkkkkkkkkkkkkkk" + "llllllllllllllll" + "mmmmmmmmmmmmmmmm" + "nnnnnnnnnnnnnnnn" + "oooooooooooooooo" + "pppppppppppppppp" + } + }; + + const size_t numTests = sizeof(pTypeTests) / sizeof(pTypeTests[0]); + size_t testIndex; + for (testIndex = 0; testIndex < numTests; ++testIndex) + { + TestSpec* spec = &pTypeTests[testIndex]; + TestState* test = constructTestSubheader(spec); + + TEST_ASSERT(doReadTest(spec, test)); + freeTestState(test); + } +#undef NUM_BANDS +#undef pixels +} + + +TEST_CASE(testTwoBandRoundTrip) +{ +#define NUM_BANDS 2 +#define pixels \ + "AaAaAaAaBbBbBbBbCcCcCcCcDdDdDdDd" \ + "AaAaAaAaBbBbBbBbCcCcCcCcDdDdDdDd" \ + "AaAaAaAaBbBbBbBbCcCcCcCcDdDdDdDd" \ + "AaAaAaAaBbBbBbBbCcCcCcCcDdDdDdDd" \ + "EeEeEeEeFfFfFfFfGgGgGgGgHhHhHhHh" \ + "EeEeEeEeFfFfFfFfGgGgGgGgHhHhHhHh" \ + "EeEeEeEeFfFfFfFfGgGgGgGgHhHhHhHh" \ + "EeEeEeEeFfFfFfFfGgGgGgGgHhHhHhHh" \ + "IiIiIiIiJjJjJjJjKkKkKkKkLlLlLlLl" \ + "IiIiIiIiJjJjJjJjKkKkKkKkLlLlLlLl" \ + "IiIiIiIiJjJjJjJjKkKkKkKkLlLlLlLl" \ + "IiIiIiIiJjJjJjJjKkKkKkKkLlLlLlLl" \ + "MmMmMmMmNnNnNnNnOoOoOoOoPpPpPpPp" \ + "MmMmMmMmNnNnNnNnOoOoOoOoPpPpPpPp" \ + "MmMmMmMmNnNnNnNnOoOoOoOoPpPpPpPp" \ + "MmMmMmMmNnNnNnNnOoOoOoOoPpPpPpPp" + + TestSpec spec = + { + "P", + 8, + pixels, + sizeof(pixels), + NUM_BANDS, + + 0, NUM_ROWS, + 0, NUM_COLS, + + "AAAAAAAAAAAAAAAA" + "BBBBBBBBBBBBBBBB" + "CCCCCCCCCCCCCCCC" + "DDDDDDDDDDDDDDDD" + "EEEEEEEEEEEEEEEE" + "FFFFFFFFFFFFFFFF" + "GGGGGGGGGGGGGGGG" + "HHHHHHHHHHHHHHHH" + "IIIIIIIIIIIIIIII" + "JJJJJJJJJJJJJJJJ" + "KKKKKKKKKKKKKKKK" + "LLLLLLLLLLLLLLLL" + "MMMMMMMMMMMMMMMM" + "NNNNNNNNNNNNNNNN" + "OOOOOOOOOOOOOOOO" + "PPPPPPPPPPPPPPPP" + "aaaaaaaaaaaaaaaa" + "bbbbbbbbbbbbbbbb" + "cccccccccccccccc" + "dddddddddddddddd" + "eeeeeeeeeeeeeeee" + "ffffffffffffffff" + "gggggggggggggggg" + "hhhhhhhhhhhhhhhh" + "iiiiiiiiiiiiiiii" + "jjjjjjjjjjjjjjjj" + "kkkkkkkkkkkkkkkk" + "llllllllllllllll" + "mmmmmmmmmmmmmmmm" + "nnnnnnnnnnnnnnnn" + "oooooooooooooooo" + "pppppppppppppppp" + }; + + TestState* test = constructTestSubheader(&spec); + TestSpec* pSpec = &spec; + + TEST_ASSERT(roundTripTest(pSpec, test, pixels)); + +#undef NUM_BANDS +#undef pixels + freeTestState(test); +} + +TEST_CASE(testPBlockOffsetBand) +{ +#define NUM_BANDS 2 +#define pixels \ + "AaAaAaAaBbBbBbBbCcCcCcCcDdDdDdDd" \ + "AaAaAaAaBbBbBbBbCcCcCcCcDdDdDdDd" \ + "AaAaAaAaBbBbBbBbCcCcCcCcDdDdDdDd" \ + "AaAaAaAaBbBbBbBbCcCcCcCcDdDdDdDd" \ + "EeEeEeEeFfFfFfFfGgGgGgGgHhHhHhHh" \ + "EeEeEeEeFfFfFfFfGgGgGgGgHhHhHhHh" \ + "EeEeEeEeFfFfFfFfGgGgGgGgHhHhHhHh" \ + "EeEeEeEeFfFfFfFfGgGgGgGgHhHhHhHh" \ + "IiIiIiIiJjJjJjJjKkKkKkKkLlLlLlLl" \ + "IiIiIiIiJjJjJjJjKkKkKkKkLlLlLlLl" \ + "IiIiIiIiJjJjJjJjKkKkKkKkLlLlLlLl" \ + "IiIiIiIiJjJjJjJjKkKkKkKkLlLlLlLl" \ + "MmMmMmMmNnNnNnNnOoOoOoOoPpPpPpPp" \ + "MmMmMmMmNnNnNnNnOoOoOoOoPpPpPpPp" \ + "MmMmMmMmNnNnNnNnOoOoOoOoPpPpPpPp" \ + "MmMmMmMmNnNnNnNnOoOoOoOoPpPpPpPp" + TestSpec pTypeTests[] = + { + + { + "P", + 8, + pixels, + sizeof(pixels), + NUM_BANDS, + + 0, 2, + 4, 4, + + "aaaabbbb" + }, + + { + "P", + 8, + pixels, + sizeof(pixels), + NUM_BANDS, + + 0, 4, + 0, NUM_COLS, + + "aaaaaaaaaaaaaaaa" + "bbbbbbbbbbbbbbbb" + "cccccccccccccccc" + "dddddddddddddddd" + }, + + { + "P", + 8, + pixels, + sizeof(pixels), + NUM_BANDS, + + 0, NUM_ROWS, + 0, NUM_COLS, + + "aaaaaaaaaaaaaaaa" + "bbbbbbbbbbbbbbbb" + "cccccccccccccccc" + "dddddddddddddddd" + "eeeeeeeeeeeeeeee" + "ffffffffffffffff" + "gggggggggggggggg" + "hhhhhhhhhhhhhhhh" + "iiiiiiiiiiiiiiii" + "jjjjjjjjjjjjjjjj" + "kkkkkkkkkkkkkkkk" + "llllllllllllllll" + "mmmmmmmmmmmmmmmm" + "nnnnnnnnnnnnnnnn" + "oooooooooooooooo" + "pppppppppppppppp" + } + }; +#undef NUM_BANDS +#undef pixels + + const size_t numTests = sizeof(pTypeTests) / sizeof(pTypeTests[0]); + size_t testIndex; + for (testIndex = 0; testIndex < numTests; ++testIndex) + { + TestSpec* spec = &pTypeTests[testIndex]; + TestState* test = constructTestSubheader(spec); + + /* Adjust subwindow to only read second band */ + test->subwindow->numBands = 1; + test->subwindow->bandList = &test->bandList[1]; + + TEST_ASSERT(doReadTest(spec, test)); + freeTestState(test); + } +} + + +TEST_CASE(testPBlockThreeBandsWithOffset) +{ +#define NUM_BANDS 3 +#define pixels \ + "Aa1Aa1Aa1Aa1Bb2Bb2Bb2Bb2Cc3Cc3Cc3Cc3Dd4Dd4Dd4Dd4" \ + "Aa1Aa1Aa1Aa1Bb2Bb2Bb2Bb2Cc3Cc3Cc3Cc3Dd4Dd4Dd4Dd4" \ + "Aa1Aa1Aa1Aa1Bb2Bb2Bb2Bb2Cc3Cc3Cc3Cc3Dd4Dd4Dd4Dd4" \ + "Aa1Aa1Aa1Aa1Bb2Bb2Bb2Bb2Cc3Cc3Cc3Cc3Dd4Dd4Dd4Dd4" \ + "Ee5Ee5Ee5Ee5Ff6Ff6Ff6Ff6Gg7Gg7Gg7Gg7Hh8Hh8Hh8Hh8" \ + "Ee5Ee5Ee5Ee5Ff6Ff6Ff6Ff6Gg7Gg7Gg7Gg7Hh8Hh8Hh8Hh8" \ + "Ee5Ee5Ee5Ee5Ff6Ff6Ff6Ff6Gg7Gg7Gg7Gg7Hh8Hh8Hh8Hh8" \ + "Ee5Ee5Ee5Ee5Ff6Ff6Ff6Ff6Gg7Gg7Gg7Gg7Hh8Hh8Hh8Hh8" \ + "Ii9Ii9Ii9Ii9Jj0Jj0Jj0Jj0Kk!Kk!Kk!Kk!Ll@Ll@Ll@Ll@" \ + "Ii9Ii9Ii9Ii9Jj0Jj0Jj0Jj0Kk!Kk!Kk!Kk!Ll@Ll@Ll@Ll@" \ + "Ii9Ii9Ii9Ii9Jj0Jj0Jj0Jj0Kk!Kk!Kk!Kk!Ll@Ll@Ll@Ll@" \ + "Ii9Ii9Ii9Ii9Jj0Jj0Jj0Jj0Kk!Kk!Kk!Kk!Ll@Ll@Ll@Ll@" \ + "Mm#Mm#Mm#Mm#Nn$Nn$Nn$Nn$Oo%Oo%Oo%Oo%Pp^Pp^Pp^Pp^" \ + "Mm#Mm#Mm#Mm#Nn$Nn$Nn$Nn$Oo%Oo%Oo%Oo%Pp^Pp^Pp^Pp^" \ + "Mm#Mm#Mm#Mm#Nn$Nn$Nn$Nn$Oo%Oo%Oo%Oo%Pp^Pp^Pp^Pp^" \ + "Mm#Mm#Mm#Mm#Nn$Nn$Nn$Nn$Oo%Oo%Oo%Oo%Pp^Pp^Pp^Pp^" + + TestSpec pTypeTests[] = + { + { + "P", + 8, + pixels, + sizeof(pixels), + NUM_BANDS, + + 0, 2, + 4, 4, + + "aaaabbbb11112222" + }, + + { + "P", + 8, + pixels, + sizeof(pixels), + NUM_BANDS, + + 0, 4, + 0, NUM_COLS, + + "aaaaaaaaaaaaaaaa" + "bbbbbbbbbbbbbbbb" + "cccccccccccccccc" + "dddddddddddddddd" + "1111111111111111" + "2222222222222222" + "3333333333333333" + "4444444444444444" + }, + + { + "P", + 8, + pixels, + sizeof(pixels), + NUM_BANDS, + + 0, NUM_ROWS, + 0, NUM_COLS, + + "aaaaaaaaaaaaaaaa" + "bbbbbbbbbbbbbbbb" + "cccccccccccccccc" + "dddddddddddddddd" + "eeeeeeeeeeeeeeee" + "ffffffffffffffff" + "gggggggggggggggg" + "hhhhhhhhhhhhhhhh" + "iiiiiiiiiiiiiiii" + "jjjjjjjjjjjjjjjj" + "kkkkkkkkkkkkkkkk" + "llllllllllllllll" + "mmmmmmmmmmmmmmmm" + "nnnnnnnnnnnnnnnn" + "oooooooooooooooo" + "pppppppppppppppp" + "1111111111111111" + "2222222222222222" + "3333333333333333" + "4444444444444444" + "5555555555555555" + "6666666666666666" + "7777777777777777" + "8888888888888888" + "9999999999999999" + "0000000000000000" + "!!!!!!!!!!!!!!!!" + "@@@@@@@@@@@@@@@@" + "################" + "$$$$$$$$$$$$$$$$" + "%%%%%%%%%%%%%%%%" + "^^^^^^^^^^^^^^^^" + } + }; +#undef NUM_BANDS +#undef pixels + + const size_t numTests = sizeof(pTypeTests) / sizeof(pTypeTests[0]); + size_t testIndex; + for (testIndex = 0; testIndex < numTests; ++testIndex) + { + TestSpec* spec = &pTypeTests[testIndex]; + TestState* test = constructTestSubheader(spec); + + test->subwindow->numBands = 2; + test->subwindow->bandList = &test->bandList[1]; + + TEST_ASSERT(doReadTest(spec, test)); + freeTestState(test); + } +} + +TEST_CASE(testInvalidReadOrderFailsGracefully) +{ +#define NUM_BANDS 3 +#define pixels \ + "Aa1Aa1Aa1Aa1Bb2Bb2Bb2Bb2Cc3Cc3Cc3Cc3Dd4Dd4Dd4Dd4"\ + "Aa1Aa1Aa1Aa1Bb2Bb2Bb2Bb2Cc3Cc3Cc3Cc3Dd4Dd4Dd4Dd4"\ + "Aa1Aa1Aa1Aa1Bb2Bb2Bb2Bb2Cc3Cc3Cc3Cc3Dd4Dd4Dd4Dd4"\ + "Aa1Aa1Aa1Aa1Bb2Bb2Bb2Bb2Cc3Cc3Cc3Cc3Dd4Dd4Dd4Dd4"\ + "Ee5Ee5Ee5Ee5Ff6Ff6Ff6Ff6Gg7Gg7Gg7Gg7Hh8Hh8Hh8Hh8"\ + "Ee5Ee5Ee5Ee5Ff6Ff6Ff6Ff6Gg7Gg7Gg7Gg7Hh8Hh8Hh8Hh8"\ + "Ee5Ee5Ee5Ee5Ff6Ff6Ff6Ff6Gg7Gg7Gg7Gg7Hh8Hh8Hh8Hh8"\ + "Ee5Ee5Ee5Ee5Ff6Ff6Ff6Ff6Gg7Gg7Gg7Gg7Hh8Hh8Hh8Hh8"\ + "Ii9Ii9Ii9Ii9Jj0Jj0Jj0Jj0Kk!Kk!Kk!Kk!Ll@Ll@Ll@Ll@"\ + "Ii9Ii9Ii9Ii9Jj0Jj0Jj0Jj0Kk!Kk!Kk!Kk!Ll@Ll@Ll@Ll@"\ + "Ii9Ii9Ii9Ii9Jj0Jj0Jj0Jj0Kk!Kk!Kk!Kk!Ll@Ll@Ll@Ll@"\ + "Ii9Ii9Ii9Ii9Jj0Jj0Jj0Jj0Kk!Kk!Kk!Kk!Ll@Ll@Ll@Ll@"\ + "Mm#Mm#Mm#Mm#Nn$Nn$Nn$Nn$Oo%Oo%Oo%Oo%Pp^Pp^Pp^Pp^"\ + "Mm#Mm#Mm#Mm#Nn$Nn$Nn$Nn$Oo%Oo%Oo%Oo%Pp^Pp^Pp^Pp^"\ + "Mm#Mm#Mm#Mm#Nn$Nn$Nn$Nn$Oo%Oo%Oo%Oo%Pp^Pp^Pp^Pp^"\ + "Mm#Mm#Mm#Mm#Nn$Nn$Nn$Nn$Oo%Oo%Oo%Oo%Pp^Pp^Pp^Pp^"\ + + nitf_Error error; + TestSpec spec = + { + "P", + 8, + pixels, + sizeof(pixels), + NUM_BANDS, + + 0, 2, + 4, 4, + + "" + }; +#undef NUM_BANDS +#undef pixels + + TestState* test = constructTestSubheader(&spec); + + test->subwindow->numBands = 2; + test->subwindow->bandList = &test->bandList[1]; + test->subwindow->bandList[0] = 2; + test->subwindow->bandList[1] = 0; + + int padded; + nitf_Uint8* user[2] = { 0, 0 }; + TEST_ASSERT(!nitf_ImageIO_read(test->imageIO, test->interface, test->subwindow, + user, &padded, &error)); + freeTestState(test); +} + +TEST_CASE(testPBlock4BytePixels) +{ +#define NUM_BANDS 2 +#define pixels \ + "**Aa**Aa**Aa**Aa**Bb**Bb**Bb**Bb**Cc**Cc**Cc**Cc**Dd**Dd**Dd**Dd"\ + "**Aa**Aa**Aa**Aa**Bb**Bb**Bb**Bb**Cc**Cc**Cc**Cc**Dd**Dd**Dd**Dd"\ + "**Aa**Aa**Aa**Aa**Bb**Bb**Bb**Bb**Cc**Cc**Cc**Cc**Dd**Dd**Dd**Dd"\ + "**Aa**Aa**Aa**Aa**Bb**Bb**Bb**Bb**Cc**Cc**Cc**Cc**Dd**Dd**Dd**Dd"\ + "**Ee**Ee**Ee**Ee**Ff**Ff**Ff**Ff**Gg**Gg**Gg**Gg**Hh**Hh**Hh**Hh"\ + "**Ee**Ee**Ee**Ee**Ff**Ff**Ff**Ff**Gg**Gg**Gg**Gg**Hh**Hh**Hh**Hh"\ + "**Ee**Ee**Ee**Ee**Ff**Ff**Ff**Ff**Gg**Gg**Gg**Gg**Hh**Hh**Hh**Hh"\ + "**Ee**Ee**Ee**Ee**Ff**Ff**Ff**Ff**Gg**Gg**Gg**Gg**Hh**Hh**Hh**Hh"\ + "**Ii**Ii**Ii**Ii**Jj**Jj**Jj**Jj**Kk**Kk**Kk**Kk**Ll**Ll**Ll**Ll"\ + "**Ii**Ii**Ii**Ii**Jj**Jj**Jj**Jj**Kk**Kk**Kk**Kk**Ll**Ll**Ll**Ll"\ + "**Ii**Ii**Ii**Ii**Jj**Jj**Jj**Jj**Kk**Kk**Kk**Kk**Ll**Ll**Ll**Ll"\ + "**Ii**Ii**Ii**Ii**Jj**Jj**Jj**Jj**Kk**Kk**Kk**Kk**Ll**Ll**Ll**Ll"\ + "**Mm**Mm**Mm**Mm**Nn**Nn**Nn**Nn**Oo**Oo**Oo**Oo**Pp**Pp**Pp**Pp"\ + "**Mm**Mm**Mm**Mm**Nn**Nn**Nn**Nn**Oo**Oo**Oo**Oo**Pp**Pp**Pp**Pp"\ + "**Mm**Mm**Mm**Mm**Nn**Nn**Nn**Nn**Oo**Oo**Oo**Oo**Pp**Pp**Pp**Pp"\ + "**Mm**Mm**Mm**Mm**Nn**Nn**Nn**Nn**Oo**Oo**Oo**Oo**Pp**Pp**Pp**Pp"\ + + TestSpec pTypeTests[] = + { + { + "P", + 16, + pixels, + sizeof(pixels), + NUM_BANDS, + + 0, NUM_ROWS, + 0, NUM_COLS, + + "aAaAaAaAaAaAaAaAaAaAaAaAaAaAaAaA" + "bBbBbBbBbBbBbBbBbBbBbBbBbBbBbBbB" + "cCcCcCcCcCcCcCcCcCcCcCcCcCcCcCcC" + "dDdDdDdDdDdDdDdDdDdDdDdDdDdDdDdD" + "eEeEeEeEeEeEeEeEeEeEeEeEeEeEeEeE" + "fFfFfFfFfFfFfFfFfFfFfFfFfFfFfFfF" + "gGgGgGgGgGgGgGgGgGgGgGgGgGgGgGgG" + "hHhHhHhHhHhHhHhHhHhHhHhHhHhHhHhH" + "iIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiI" + "jJjJjJjJjJjJjJjJjJjJjJjJjJjJjJjJ" + "kKkKkKkKkKkKkKkKkKkKkKkKkKkKkKkK" + "lLlLlLlLlLlLlLlLlLlLlLlLlLlLlLlL" + "mMmMmMmMmMmMmMmMmMmMmMmMmMmMmMmM" + "nNnNnNnNnNnNnNnNnNnNnNnNnNnNnNnN" + "oOoOoOoOoOoOoOoOoOoOoOoOoOoOoOoO" + "pPpPpPpPpPpPpPpPpPpPpPpPpPpPpPpP" + } + }; + +#undef NUM_BANDS +#undef pixels + const size_t numTests = sizeof(pTypeTests) / sizeof(pTypeTests[0]); + size_t testIndex; + for (testIndex = 0; testIndex < numTests; ++testIndex) + { + TestSpec* spec = &pTypeTests[testIndex]; + TestState* test = constructTestSubheader(spec); + + /* Adjust subwindow to only read second band */ + test->subwindow->numBands = 1; + test->subwindow->bandList = &test->bandList[1]; + + TEST_ASSERT(doReadTest(spec, test)); + freeTestState(test); + } +} + +int main(int argc, char** argv) +{ + (void) argc; + (void) argv; + CHECK(testPBlockOneBand); + CHECK(testPBlockTwoBands); + CHECK(testPBlockOffsetBand); + CHECK(testPBlockThreeBandsWithOffset); + CHECK(testInvalidReadOrderFailsGracefully); + CHECK(testPBlock4BytePixels); + CHECK(testTwoBandRoundTrip); + return 0; +} diff --git a/modules/c/nrt/unittests/test_byte_swap.c b/modules/c/nrt/unittests/test_nrt_byte_swap similarity index 100% rename from modules/c/nrt/unittests/test_byte_swap.c rename to modules/c/nrt/unittests/test_nrt_byte_swap diff --git a/modules/python/nitf/source/generated/nitropy.py b/modules/python/nitf/source/generated/nitropy.py index e4af1a6cb..decf2669f 100644 --- a/modules/python/nitf/source/generated/nitropy.py +++ b/modules/python/nitf/source/generated/nitropy.py @@ -1620,8 +1620,8 @@ def nitf_ImageWriter_setWriteCaching(iWriter, enable): return _nitropy.nitf_ImageWriter_setWriteCaching(iWriter, enable) nitf_ImageWriter_setWriteCaching = _nitropy.nitf_ImageWriter_setWriteCaching -def nitf_ImageWriter_setDirectBlockWrite(iWriter, enable): - return _nitropy.nitf_ImageWriter_setDirectBlockWrite(iWriter, enable) +def nitf_ImageWriter_setDirectBlockWrite(iWriter, enable, error): + return _nitropy.nitf_ImageWriter_setDirectBlockWrite(iWriter, enable, error) nitf_ImageWriter_setDirectBlockWrite = _nitropy.nitf_ImageWriter_setDirectBlockWrite def nitf_ImageWriter_setPadPixel(imageWriter, value, length, error): diff --git a/modules/python/nitf/source/generated/nitropy_wrap.cxx b/modules/python/nitf/source/generated/nitropy_wrap.cxx index a6d8657f0..f937ea7e3 100644 --- a/modules/python/nitf/source/generated/nitropy_wrap.cxx +++ b/modules/python/nitf/source/generated/nitropy_wrap.cxx @@ -17035,14 +17035,19 @@ SWIGINTERN PyObject *_wrap_nitf_ImageWriter_setDirectBlockWrite(PyObject *SWIGUN PyObject *resultobj = 0; nitf_ImageWriter *arg1 = (nitf_ImageWriter *) 0 ; int arg2 ; + nitf_Error *arg3 = (nitf_Error *) 0 ; void *argp1 = 0 ; int res1 = 0 ; int val2 ; int ecode2 = 0 ; + void *argp3 = 0 ; + int res3 = 0 ; PyObject * obj0 = 0 ; PyObject * obj1 = 0 ; + PyObject * obj2 = 0 ; + bool result; - if (!PyArg_ParseTuple(args,(char *)"OO:nitf_ImageWriter_setDirectBlockWrite",&obj0,&obj1)) SWIG_fail; + if (!PyArg_ParseTuple(args,(char *)"OOO:nitf_ImageWriter_setDirectBlockWrite",&obj0,&obj1,&obj2)) SWIG_fail; res1 = SWIG_ConvertPtr(obj0, &argp1,SWIGTYPE_p_nitf_WriteHandler, 0 | 0 ); if (!SWIG_IsOK(res1)) { SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "nitf_ImageWriter_setDirectBlockWrite" "', argument " "1"" of type '" "nitf_ImageWriter *""'"); @@ -17053,8 +17058,13 @@ SWIGINTERN PyObject *_wrap_nitf_ImageWriter_setDirectBlockWrite(PyObject *SWIGUN SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "nitf_ImageWriter_setDirectBlockWrite" "', argument " "2"" of type '" "int""'"); } arg2 = static_cast< int >(val2); - nitf_ImageWriter_setDirectBlockWrite(arg1,arg2); - resultobj = SWIG_Py_Void(); + res3 = SWIG_ConvertPtr(obj2, &argp3,SWIGTYPE_p__NRT_Error, 0 | 0 ); + if (!SWIG_IsOK(res3)) { + SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "nitf_ImageWriter_setDirectBlockWrite" "', argument " "3"" of type '" "nitf_Error *""'"); + } + arg3 = reinterpret_cast< nitf_Error * >(argp3); + result = (bool)nitf_ImageWriter_setDirectBlockWrite(arg1,arg2,arg3); + resultobj = SWIG_From_bool(static_cast< bool >(result)); return resultobj; fail: return NULL; diff --git a/modules/wscript b/modules/wscript index d4c8631b8..142a31128 100644 --- a/modules/wscript +++ b/modules/wscript @@ -8,7 +8,10 @@ def configure(conf): conf.recurse() def build(bld): - bld.recurse() + if bld.env['LIB_TYPE'] == 'shlib': + bld.recurse('drivers c') + else: + bld.recurse() def distclean(context): context.recurse()