make.py #1 

#!/usr/bin/python

import subprocess
import os
import time
import platform

import csbuild
from csbuild import log

csbuild.Toolchain("gcc").Compiler().SetCppStandard("c++11")
csbuild.Toolchain("gcc").SetCxxCommand("clang++")

csbuild.Toolchain("gcc").Compiler().AddWarnFlags("all", "extra", "ctor-dtor-privacy", "overloaded-virtual", "init-self", "missing-include-dirs", "switch-default", "no-switch-enum", "undef", "no-old-style-cast")

csbuild.DisablePrecompile()

csbuild.AddOption("--with-mongo", action="store", help="Path to mongo include directory. If not specified, mongo will not be built.", nargs="?", default=None, const="/usr")
csbuild.AddOption("--with-boost", action="store", help="Path to boost include directory. If not specified, mongo will not be built.", nargs="?", default=None, const="/usr")
csbuild.AddOption("--no-threads", action="store_true", help="Build without thread support")
csbuild.AddOption("--no-exceptions", action="store_true", help="Build without exception support")
csbuild.AddOption("--no-unit-tests", action="store_true", help="Don't automatically run unit tests as part of build")
csbuild.SetHeaderInstallSubdirectory("sprawl/{project.name}")

csbuild.SetUserData("subdir", platform.system())

if platform.system() == "Darwin":
	csbuild.Toolchain("gcc").AddDefines("_XOPEN_SOURCE");
	csbuild.Toolchain("gcc").SetCppStandardLibrary("libc++")

csbuild.SetOutputDirectory("lib/{project.userData.subdir}/{project.activeToolchainName}/{project.outputArchitecture}/{project.targetName}")
csbuild.SetIntermediateDirectory("Intermediate/{project.userData.subdir}/{project.activeToolchainName}/{project.outputArchitecture}/{project.targetName}/{project.name}")


csbuild.Toolchain("msvc").AddCompilerFlags(
	"/fp:fast",
	"/wd\"4530\"",
	"/wd\"4067\"",
	"/wd\"4351\"",
	"/constexpr:steps1000000",
)

if not csbuild.GetOption("no_threads"):
	csbuild.Toolchain("gcc", "ios", "android").AddCompilerFlags("-pthread")

if csbuild.GetOption("no_exceptions"):
	csbuild.Toolchain("gcc", "ios", "android").AddCompilerFlags("-fno-exceptions")
else:
	csbuild.Toolchain("msvc").AddCompilerFlags("/EHsc")

	
@csbuild.project("collections", "collections")
def collections():
	csbuild.SetOutput("libsprawl_collections", csbuild.ProjectType.StaticLibrary)

	csbuild.EnableHeaderInstall()

	
@csbuild.project("tag", "tag")
def collections():
	csbuild.SetOutput("libsprawl_tag", csbuild.ProjectType.StaticLibrary)

	csbuild.EnableHeaderInstall()

@csbuild.project("if", "if")
def collections():
	csbuild.SetOutput("libsprawl_if", csbuild.ProjectType.StaticLibrary)

	csbuild.EnableHeaderInstall()


@csbuild.project("network", "network")
def network():
	csbuild.SetOutput("libsprawl_network", csbuild.ProjectType.StaticLibrary)
	
	csbuild.EnableOutputInstall()
	csbuild.EnableHeaderInstall()

	
@csbuild.project("serialization", "serialization")
def serialization():
	csbuild.SetOutput("libsprawl_serialization", csbuild.ProjectType.StaticLibrary)
	csbuild.AddExcludeDirectories("serialization/mongo")

	csbuild.EnableOutputInstall()
	csbuild.EnableHeaderInstall()

	
@csbuild.project("time", "time")
def timeProject():
	csbuild.SetOutput("libsprawl_time", csbuild.ProjectType.StaticLibrary)

	csbuild.Toolchain("gcc").AddExcludeFiles("time/*_windows.cpp")
	if platform.system() == "Darwin":
		csbuild.Toolchain("gcc").AddExcludeFiles("time/*_linux.cpp")
	else:
		csbuild.Toolchain("gcc").AddExcludeFiles("time/*_osx.cpp")

	csbuild.Toolchain("msvc").AddExcludeFiles("time/*_linux.cpp", "time/*_osx.cpp")

	csbuild.EnableOutputInstall()
	csbuild.EnableHeaderInstall()

	
@csbuild.project("filesystem", "filesystem")
def filesystem():
	csbuild.SetOutput("libsprawl_filesystem", csbuild.ProjectType.StaticLibrary)

	csbuild.Toolchain("gcc").AddExcludeFiles("filesystem/*_windows.cpp")
	csbuild.Toolchain("msvc").AddExcludeFiles("filesystem/*_linux.cpp")

	csbuild.EnableOutputInstall()
	csbuild.EnableHeaderInstall()

	
@csbuild.project("threading", "threading")
def threading():
	csbuild.SetOutput("libsprawl_threading", csbuild.ProjectType.StaticLibrary)

	if platform.system() != "Darwin":
		@csbuild.scope(csbuild.ScopeDef.Final)
		def finalScope():
			csbuild.Toolchain("gcc").Linker().AddLinkerFlags("-pthread")

	csbuild.Toolchain("gcc").AddExcludeFiles("threading/*_windows.cpp")
	if platform.system() == "Darwin":
		csbuild.Toolchain("gcc").AddExcludeFiles("threading/event_linux.cpp")
	else:
		csbuild.Toolchain("gcc").AddExcludeFiles("threading/event_osx.cpp")

	csbuild.Toolchain("msvc").AddExcludeFiles(
		"threading/*_linux.cpp",
		"threading/*_osx.cpp"
	)

	csbuild.EnableOutputInstall()
	csbuild.EnableHeaderInstall()

MongoDir = csbuild.GetOption("with_mongo")
BoostDir = csbuild.GetOption("with_boost")
if (not MongoDir) ^ (not BoostDir):
	log.LOG_ERROR("Both mongo and boost directories must be specified to build MongoSerializer.");
	csbuild.Exit(1)
	
if MongoDir and BoostDir:
	MongoDir = os.path.abspath(MongoDir)
	BoostDir = os.path.abspath(BoostDir)
	@csbuild.project("serialization-mongo", "serialization/mongo")
	def serialization():
		csbuild.SetOutput("libsprawl_serialization-mongo", csbuild.ProjectType.StaticLibrary)
		csbuild.AddDefines("BOOST_ALL_NO_LIB")
		
		csbuild.AddIncludeDirectories(
			"./serialization",
			os.path.join(MongoDir, "include"),
			os.path.join(BoostDir, "include")
		)

		csbuild.AddLibraryDirectories(
			os.path.join(MongoDir, "lib"),
			os.path.join(BoostDir, "lib")
		)
	
		csbuild.SetHeaderInstallSubdirectory("sprawl/serialization")
		csbuild.EnableOutputInstall()
		csbuild.EnableHeaderInstall()

@csbuild.project("memory", "memory")
def memory():
	csbuild.SetOutput("libsprawl_memory", csbuild.ProjectType.StaticLibrary)

	csbuild.EnableHeaderInstall()


@csbuild.project("string", "string")
def string():
	csbuild.SetOutput("libsprawl_string", csbuild.ProjectType.StaticLibrary)

	csbuild.EnableOutputInstall()
	csbuild.EnableHeaderInstall()


@csbuild.project("hash", "hash")
def hash():
	csbuild.SetOutput("libsprawl_hash", csbuild.ProjectType.StaticLibrary)

	csbuild.EnableOutputInstall()
	csbuild.EnableHeaderInstall()

@csbuild.project("logging", "logging")
def logging():
	csbuild.SetOutput("libsprawl_logging", csbuild.ProjectType.StaticLibrary)

	@csbuild.scope(csbuild.ScopeDef.Final)
	def finalScope():
		if platform.system() != "Darwin":
			csbuild.Toolchain("gcc").AddLibraries(
				"bfd",
			)
		csbuild.Toolchain("msvc").AddLibraries(
			"DbgHelp"
		)

	csbuild.Toolchain("gcc").AddExcludeFiles("logging/*_windows.cpp")
	if platform.system() == "Darwin":
		csbuild.Toolchain("gcc").AddExcludeFiles("logging/*_linux.cpp")
	else:
		csbuild.Toolchain("gcc").AddExcludeFiles("logging/*_osx.cpp")
	csbuild.Toolchain("msvc").AddExcludeFiles(
		"logging/*_linux.cpp",
		"logging/*_osx.cpp"
	)

	csbuild.EnableOutputInstall()
	csbuild.EnableHeaderInstall()

@csbuild.project("common", "common")
def common():
	csbuild.SetOutput("libsprawl_common", csbuild.ProjectType.StaticLibrary)

	csbuild.EnableHeaderInstall()

UnitTestDepends = ["serialization", "string", "hash", "time", "threading", "filesystem", "logging"]
if MongoDir:
	UnitTestDepends.append("serialization-mongo")

@csbuild.project("UnitTests", "UnitTests", UnitTestDepends)
def UnitTests():
	csbuild.DisableChunkedBuild()
	csbuild.SetOutput("SprawlUnitTest")
	csbuild.SetOutputDirectory("bin/{project.userData.subdir}/{project.activeToolchainName}/{project.outputArchitecture}/{project.targetName}")
	
	csbuild.EnableOutputInstall()
	
	csbuild.AddIncludeDirectories(
		"UnitTests/gtest",
		"UnitTests/gtest/include",
	)

	csbuild.Toolchain("gcc").Compiler().AddWarnFlags("no-undef", "no-switch-enum", "no-missing-field-initializers")

	csbuild.AddExcludeFiles(
		"UnitTests/gtest/src/gtest-death-test.cc",
		"UnitTests/gtest/src/gtest-filepath.cc",
		"UnitTests/gtest/src/gtest-internal-inl.h",
		"UnitTests/gtest/src/gtest-port.cc",
		"UnitTests/gtest/src/gtest-printers.cc",
		"UnitTests/gtest/src/gtest-test-part.cc",
		"UnitTests/gtest/src/gtest-typed-test.cc",
		"UnitTests/gtest/src/gtest.cc",
	)

	if MongoDir:
		csbuild.AddIncludeDirectories(
			"./serialization",
			os.path.join(MongoDir, "include"),
			os.path.join(BoostDir, "include")
		)

		csbuild.AddLibraryDirectories(
			os.path.join(MongoDir, "lib"),
			os.path.join(BoostDir, "lib")
		)
		
		csbuild.AddLibraries(
			"mongoclient",
			"boost_filesystem",
			"boost_system",
			"boost_thread",
			"boost_program_options",
			"ssl",
			"crypto",
		)
		csbuild.Toolchain("gcc").AddLibraries("pthread")
		csbuild.Toolchain("gcc").AddCompilerFlags("-pthread")
		csbuild.AddDefines("WITH_MONGO")
	else:
		csbuild.AddExcludeFiles(
			"UnitTests/UnitTests_MongoReplicable.cpp",
		)
# Change User Description Committed
#1 23398 ququlala "Forking branch Mainline of shadauxcat-libsprawl to ququlala-libsprawl."
//guest/ShadauxCat/Sprawl/Mainline/make.py
#17 20951 ShadauxCat - Reworked ConcurrentQueue::Dequeue to be a lot faster and wait-free
- Added constexpr constructor for sprawl tag
- Added support for specifying base for int-to-tag conversion
- Added ability to force poolallocator to pass through to malloc
- Added a helper class that will eventually be used in StringBuilder for faster int-to-string conversion

#review-20952 post-commit
#16 19906 ShadauxCat - Added tag, compile time string type
- Since tag requires visual studio 2015, removed compatibility code for earlier versions of visual studio
- Improved compiler detection
- Added endianness detection
- Added template if/else helper
- Fixed bug with murmur3 64 bit
- Added seed argument for murmur3

#review-19907
#15 16768 ShadauxCat Improvements to error handling in builds with exceptions disabled:

- In debug builds or with SPRAWL_ERRORSTATE_STRICT enabled, ErrorState will output a message to stderr and terminate if Get() is called when an error flag is set. (In release buils or with SPRAWL_ERRORSTATE_PERMISSIVE defined, Get() will return junk memory in this case.)
- In debug builds or with SPRAWL_ERRORSTATE_STRICT enabled, ErrorState will output a message to stderr and terminate if its destructor is called without checking the errorstate if an error is present (equivalent to an exception terminating the application if no catch() block is present for it).
- On linux builds and when running "Analyze" through visual studio, a warning will be issued if any function returning ErrorState has its return value ignored. (This only applies to builds with exceptions not enabled; when exceptions are enabled no warning is issued)
- Many functions that could return ErrorState were having their return values silently ignored in internal sprawl code so the user would not find out about errors if exceptions are disabled; now anything in sprawl code that calls a function returning ErrorState will either handle the error, or (in most cases) surface it back up to the user.
- As a positive side-effect of the warnings for ignoring ErrorState, several constructors that were capable of throwing exceptions are no longer capable of doing so.

#review-16769
#14 16283 ShadauxCat Changed linux backtrace library to use backtrace() instead of libunwind.
backtrace() costs about 1 us more per call than libunwind does, but saves a dependency, so that extra 1us seems not to matter much in the face of external dependency requirements. This ensures the only external dependency is libbfd, which is more readily available on more systems than libunwind and also more likely to already be installed.

libunwind's major performance benefit over glibc is in symbolification, and we don't use libunwind for symbolification because libbfd gives us more information (i.e., filename and line numbers). So the dependency has very very little real benefit.

#review-16284
#13 16175 ShadauxCat Enabled exceptions on windows release build...

#review-16176
#12 16052 ShadauxCat - Changed default block size for concurrent queue to a more reasonable value
- Changed some memory orders to memory_order_seq_cst when they don't actually need to be that to get around a bug in visual studio 2013 - debug builds assert when memory_order_acq_rel is used for a compare_exchange_strong (this is a standard library bug and is fixed in VS2015)
- Added Event API - events are an alternative to condition variables that do not require a mutex and are guaranteed not to miss any signals, even if the signal comes while the thread is not listening for it. Unlike condition variables, however, they do not support broadcasting (and in fact, in general, are not safe to use with multiple threads listening for the same event simultaneously - though notifying on the same event is fine)
- Rewrote ThreadManager around ConcurrentQueue and Event API so it is now lock-free. Also improved some behaviors of the staged thread manager operation so it now supports tasks that can be run on multiple stages via a bitmask.
- Fixed an issue where the Coroutine copy constructor was calling the std::function constructor instead and another where initializing with a stack might try to call the wrong constructor and vice-versa
- Fixed Coroutine never calling munmap() on its stack in linux and causing a memory leak
- Added default arguments to time functions
- Attempted to fix some issues with BinaryTree. Fixed some but not all. It's currently not suitable for use, sadly.
- Logging Improvements:
- - Added thread ID to logging
- - Fixed some issues with category handlers
- - Added backtraces
- - Added the following additional log macros:
- - - LOG_IF
- - - LOG_EVERY_N
- - - LOG_FIRST_N
- - - LOG_IF_EVERY_N
- - - LOG_IF_FIRST_N
- - - LOG_ASSERT
- - Added the ability to set extra info callbacks to get data such as script backtraces
- - Removed the thread-related handlers and replaced them with RunHandler_Threaded and RunHandler_ThreadManager, which will enable any passed-in handler to be run in a threaded fashion
- Removed StaticPoolAllocator and renamed DynamicPoolAllocator to PoolAllocator; adjusted unit tests accordingly
- PoolAllocator now allocates its pool with mmap and VirtualAlloc, rather than with malloc
- Fixed a bug with Vector copy assignment operator
- Improved performance of StringBuilder considerably for cases where there are no modifier strings
- Removed Copy-On-Write behavior of JSONToken as it was broken; copies are now performed with explicit DeepCopy() and ShallowCopy() functions
- Fixed some parser bugs with JSONToken
- Added iteration to JSONToken to iterate its children
- Fixed crash when reading a negative number of bytes from a file
- Changed StringBuilder to favor speed instead of memory by default
- Added some performance unit tests for JSON token

#review-16053
#11 15891 ShadauxCat Concurrent queue implementation, plus updates to makefile in light of recent csbuild changes.

#review-15892
#10 14833 ShadauxCat First checkin of logging module.

Also fixes the following issues:

-Added UpperBound() and LowerBound() to BinaryTree and created appropriate unit tests
-Added Sync() to ThreadManager to force it to run all tasks to completion and not return until it has no tasks left
-Fixed a bug in String::format() where a non-numeric value inside {} would be treated as an empty {}; it now simply prints whatever the value was. (i.e., "{blah}".format(foo) simply returns "{blah}")
-Added Reset() to sprawl::StringBuilder
-Disabled the switch-enum warning flag in gcc because it's stupid and ridiculous that a default case doesn't shut it up
-Made sprawl::Mutex movable. This may turn out to be a bad idea but it enabled keeping them in a map.
-Fixed a name collission between HashMap and BinaryTree; both defined sprawl::collections::detail::UnderlyingType and ::MethodType. Prefixed the ones in BinaryTree with "Tree". This isn't the best solution, but it works for now.

#review-14834
#9 14761 ShadauxCat First drop of code for sprawl::filesystem and sprawl::path.
Library will continue to grow.

Also fixed a warning on linux.

#review-14762
#8 14218 ShadauxCat Fixed makefile issue - windows incorrectly compiling an osx file.
#7 14216 ShadauxCat -Moved some global sprawl::Strings into local scope in json serialization test because of initialization order issues in the memory allocator on mac.
This is a temporary fix, and a real fix will come by making the pool allocator work in explicitly-sized pieces and putting all the code for those pieces into a cpp file.
-Fixed a large number of warnings on mac/linux that were exposed by fixes to csbuild
-Fixed compile errors on mac due to malloc and alloca not being defined, fixed by #include <stdlib.h> in appropriate places
-Fixed mac os x trying to link against pthread erroneously
-Provided os x implementation of time library
-Fixed compile errors on os x due to std::unordered_map whining about the difference between an allocator that allocates std::pair<key, value> and one that allocates std::pair<key const, value>, which, of course, is that the allocator will be no different at all.
-Fixed an actual issue where one unordered_map was allocating only key_type instead of std::pair<key_type, value_type>
-Fixed a memory leak where coroutine objects would never be cleaned up because either Yield() or reactivate_() will never return (and thus never clean up their stack memory and thus never release any dynamic memory held in stack objects) depending on the situation - if the function runs to completion, reactivate_() never returns after calling swapcontext(); meanwhile, if the function does not run to completion, Yield() never returns after calling Pause(). This behavior will need to be well-documented because it will affect client-side code as well. Stack memory within a coroutine should not rely on RAII behavior.
-Fixed compile failure when creating a StlWrapper with a const value_type

#review-14217
#6 14202 ShadauxCat Added option to build without running unit tests.

#review-14203
#5 14144 ShadauxCat Switching unit tests to gtest.
100 is a decent number of tests to start with, but it needs to be more like 400 to test the current codebase.

#review-14145
#4 13660 ShadauxCat Fixed a makefile error - unit tests were being run from the wrong location.

#review-13661
#3 13650 ShadauxCat - Windows implementations of thread and time libraries
- Added coroutines
- Added some more unit tests, fixed some unit tests in windows environments
- Fixed an issue where multi threading was not properly detected on Linux
- Fixed the makefiles to build with threading by default on linux
- Changed the pool allocator to use thread-local pools instead of locking mutexes
- Fixed output of sprawl::string in the StringBuilder library to take length into account
- Added string builder options for StringLiteral
- Added thread local implementation

#review
#2 12508 ShadauxCat -Added threading library.
Currently only functional for Linux; Windows will fail to link. (I will fix this soon.)
-Fixed missing move and copy constructors in List and ForwardList
-Fixed broken move constructor in HashMap
-Fixed missing const get() in HashMap
-Fixed broken operator-> in ListIterator
-Added sprawl::noncopyable
-Added sketch headers for filesystem library
-Made StringLiteral hashable, added special hashes for pointers and integers in murmur3
-Fixed compiler warning in async_network
-Updated memory allocators to use new threading library for mutexes
-Added accessibility to sprawl::StringLiteral to be able toa ccess its pointer and length and perform pointer comparisons

#review-12504
#1 11496 ShadauxCat Initial checkin: Current states for csbuild and libSprawl