add functional c tests via check-enzyme-c command (#15)

* add functional c tests via check-enzyme-c command

* add functional c tests to github workflows

* fix hardcoded path

* test paths again

* another attempt at fix

* another attempt to fix clang path in ci

* install clang

* test loops in build system

* another build test

* change return type of externed builtinautodiff to fix build

* think it works with build system now?

* Add an expected fail test

* make ifcrash.ll be expect fail

* make tests use clang instead of clang++ for c

* only run functional tests on llvm 7

.github/workflows/functional_tests.yml

@@ -0,0 +1,32 @@
+name: Functional Tests CI
+
+on: [push]
+
+jobs:
+  build:
+    name: Test on os ${{ matrix.os }} and llvm ${{ matrix.llvm }} mode ${{ matrix.build }}
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        llvm: ["7"]
+        build: ["Release", "Debug"] # "RelWithDebInfo"
+        os: [ubuntu-18.04]
+
+    steps:
+    - name: add llvm 
+      run: sudo apt-get install -y llvm-${{ matrix.llvm }}-dev llvm-${{ matrix.llvm }}-tools clang-${{ matrix.llvm }}
+    - uses: actions/checkout@v1
+      with:
+          fetch-depth: 1
+    - name: mkdir
+      run: cd enzyme && mkdir build
+    - name: cmake
+      run: |
+          cd enzyme/build 
+          cmake .. -DLLVM_EXTERNAL_LIT=/usr/lib/llvm-${{ matrix.llvm }}/build/utils/lit/lit.py -DCMAKE_BUILD_TYPE=${{ matrix.build }} -DLLVM_DIR=/usr/lib/llvm-${{ matrix.llvm }}/lib/cmake/llvm
+    - name: make
+      run: cd enzyme/build && make
+    - name: make check-enzyme-c
+      run: cd enzyme/build && make check-enzyme-c

enzyme/CMakeLists.txt

@@ -48,3 +48,4 @@ message("found llvm version " ${LLVM_VERSION_MAJOR})
 
 add_subdirectory(Enzyme)
 add_subdirectory(test)
+add_subdirectory(functional_tests_c)

enzyme/functional_tests_c/.gitignore

@@ -0,0 +1 @@
+build/*

enzyme/functional_tests_c/CMakeLists.txt

@@ -0,0 +1,24 @@
+configure_lit_site_cfg(
+  ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
+  ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py
+  MAIN_CONFIG
+  ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py
+)
+
+set(ENZYME_TEST_DEPS LLVMEnzyme-${LLVM_VERSION_MAJOR})
+
+set(ENZYME_TESTSUITES)
+list(APPEND ${CMAKE_CURRENT_BINARY_DIR}/lit.cfg)
+
+# Run regression and unit tests
+add_lit_testsuite(check-enzyme-c "Running enzyme regression tests"
+    ${CMAKE_CURRENT_BINARY_DIR}
+    DEPENDS ${ENZYME_TEST_DEPS}
+    ARGS -v
+)
+
+set_target_properties(check-enzyme-c PROPERTIES FOLDER "Tests")
+
+#add_lit_testsuites(ENZYME ${CMAKE_CURRENT_SOURCE_DIR}
+    #    DEPENDS ${ENZYME_TEST_DEPS}
+#)

enzyme/functional_tests_c/FAIL_insertsort_sum_alt.c

@@ -0,0 +1,99 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <math.h>
+#include <assert.h>
+
+#define __builtin_autodiff __enzyme_autodiff
+double __enzyme_autodiff(void*, ...);
+//float man_max(float* a, float* b) {
+//  if (*a > *b) {
+//    return *a;
+//  } else {
+//    return *b;
+//  }
+//}
+
+
+// size of array
+float* unsorted_array_init(int N) {
+  float* arr = (float*) malloc(sizeof(float) * N);
+  for (int i = 0; i < N; i++) {
+    arr[i] = 1.0*(i%2);
+  }
+  return arr;
+}
+
+__attribute__((noinline))
+void insertion_sort_inner(float* array, int i) {
+    int j = i;
+    while (j > 0 && array[j-1] > array[j]) {
+      float tmp = array[j];
+      array[j] = array[j-1];
+      array[j-1] = tmp;
+      j -= 1;
+    }
+}
+
+// sums the first half of a sorted array.
+void insertsort_sum (float* array, int N, float* ret) {
+  float sum = 0;
+  //qsort(array, N, sizeof(float), cmp);
+
+  for (int i = 1; i < N; i++) {
+    insertion_sort_inner(array, i);
+  }
+
+
+  for (int i = 0; i < N/2; i++) {
+    //printf("Val: %f\n", array[i]);
+    sum += array[i];
+  }
+  *ret = sum;
+}
+
+
+
+
+int main(int argc, char** argv) {
+
+
+
+  float a = 2.0;
+  float b = 3.0;
+
+
+
+  float da = 0;//(float*) malloc(sizeof(float));
+  float db = 0;//(float*) malloc(sizeof(float));
+
+
+  float ret = 0;
+  float dret = 1.0;
+
+  int N = 10;
+  int dN = 0;
+  float* array = unsorted_array_init(N);
+  float* d_array = (float*) malloc(sizeof(float)*N);
+  for (int i = 0; i < N; i++) {
+    d_array[i] = 0.0;
+  }
+
+  printf("The total sum is %f\n", ret);
+
+  __builtin_autodiff(insertsort_sum, array, d_array, N, &ret, &dret);
+
+  for (int i = 0; i < N; i++) {
+    printf("Diffe for index %d is %f\n", i, d_array[i]);
+    if (i%2 == 0) {
+      assert(d_array[i] == 0.0);
+    } else {
+      assert(d_array[i] == 1.0);
+    }
+  }
+
+  //assert(da == 100*1.0f);
+  //assert(db == 100*1.0f);
+
+  //printf("hello! %f, res2 %f, da: %f, db: %f\n", ret, ret, da,db);
+  return 0;
+}

enzyme/functional_tests_c/Makefile

@@ -0,0 +1,88 @@
+
+.PHONY:	all clean
+
+OBJ := $(wildcard *.c)
+
+#ENZYME_PLUGIN := ./../build/Enzyme/LLVMEnzyme-7.so
+#export PATH = "$(PATH):$(LLVM_DIR)"
+
+#ifeq ($(CLANG_BIN_PATH),)
+#CLANG_BIN_PATH=${CLANG_BIN_PATH}
+#endif
+#
+#ifeq ($(ENZYME_PLUGIN),)
+#ENZYME_PLUGIN=${ENZYME_PLUGIN}
+#endif
+
+#CLANG_BIN_PATH := 
+
+all: $(patsubst %.c,build/%-enzyme0,$(OBJ)) $(patsubst %.c,build/%-enzyme1,$(OBJ)) $(patsubst %.c,build/%-enzyme2,$(OBJ)) $(patsubst %.c,build/%-enzyme3,$(OBJ))
+
+POST_ENZYME_FLAGS := -mem2reg -sroa -adce -simplifycfg
+
+#all: $(patsubst %.c,build/%-enzyme1,$(OBJ)) $(patsubst %.c,build/%-enzyme2,$(OBJ)) $(patsubst %.c,build/%-enzyme3,$(OBJ))
+#clean:
+#	rm -f main main-* main.ll
+#	rm -f compilercrash compilercrash-* compilercrash.ll
+#	rm -f segfault segfault-* segfault.ll
+#	rm -f silent_failure silent_failure-* silent_failure.ll
+
+#ENZYME_PLUGIN = $(ENZYME_PLUGIN)
+
+#EXTRA_FLAGS = -indvars -loop-simplify -loop-rotate
+
+# NOTE(TFK): Optimization level 0 is broken right now.
+build/%-enzyme0: %.c
+	@./setup.sh $(CLANG_BIN_PATH)/clang -std=c11 -O1 $(patsubst %.c,%,$<).c -S -emit-llvm -o $@.ll
+	@./setup.sh $(CLANG_BIN_PATH)/opt $@.ll $(EXTRA_FLAGS) -load=$(ENZYME_PLUGIN) -enzyme -o $@.bc
+	@./setup.sh $(CLANG_BIN_PATH)/clang -std=c11 $@.bc -S -emit-llvm -o $@-final.ll
+	@./setup.sh $(CLANG_BIN_PATH)/clang -std=c11 $@.bc -o $@
+
+build/%-enzyme1: %.c
+	@./setup.sh $(CLANG_BIN_PATH)/clang -std=c11 -O1 $(patsubst %.c,%,$<).c -S -emit-llvm -o $@.ll
+	@./setup.sh $(CLANG_BIN_PATH)/opt $@.ll $(EXTRA_FLAGS) -load=$(ENZYME_PLUGIN) -enzyme -o $@.bc
+	@./setup.sh $(CLANG_BIN_PATH)/clang -std=c11 $@.bc -S -emit-llvm -o $@-final.ll
+	@./setup.sh $(CLANG_BIN_PATH)/clang -std=c11 $@.bc -o $@
+
+build/%-enzyme2: %.c
+	@./setup.sh $(CLANG_BIN_PATH)/clang -std=c11 -O2 $(patsubst %.c,%,$<).c -S -emit-llvm -o $@.ll
+	@./setup.sh $(CLANG_BIN_PATH)/opt $@.ll $(EXTRA_FLAGS) -load=$(ENZYME_PLUGIN) -enzyme -o $@.bc
+	@./setup.sh $(CLANG_BIN_PATH)/clang -std=c11 $@.bc -S -emit-llvm -o $@-final.ll
+	@./setup.sh $(CLANG_BIN_PATH)/clang -std=c11 $@.bc -o $@
+
+build/%-enzyme3: %.c
+	@./setup.sh $(CLANG_BIN_PATH)/clang -std=c11 -O3 $(patsubst %.c,%,$<).c -S -emit-llvm -o $@.ll
+	@./setup.sh $(CLANG_BIN_PATH)/opt $@.ll $(EXTRA_FLAGS) -load=$(ENZYME_PLUGIN) -enzyme $(POST_ENZYME_FLAGS) -o $@.bc
+	@./setup.sh $(CLANG_BIN_PATH)/clang -std=c11 $@.bc -S -emit-llvm -o $@-final.ll
+	@./setup.sh $(CLANG_BIN_PATH)/clang -std=c11 $@.bc -o $@
+
+
+%-enzyme-test0: build/%-enzyme0
+	@./$< 2> /dev/null 1> /dev/null && echo "success" || echo "FAILURE"
+
+%-enzyme-test1: build/%-enzyme1
+	@./$< 2> /dev/null 1> /dev/null && echo "success" || echo "FAILURE"
+
+%-enzyme-test2: build/%-enzyme2
+	@./$< 2> /dev/null 1> /dev/null && echo "success" || echo "FAILURE"
+
+%-enzyme-test3: build/%-enzyme3
+	@./$< 2> /dev/null 1> /dev/null && echo "success" || echo "FAILURE"
+
+test: $(patsubst %.c,%-enzyme-test0,$(OBJ)) $(patsubst %.c,%-enzyme-test1,$(OBJ)) $(patsubst %.c,%-enzyme-test2,$(OBJ)) $(patsubst %.c,%-enzyme-test3,$(OBJ))
+#test: $(patsubst %.c,%-enzyme-test1,$(OBJ)) $(patsubst %.c,%-enzyme-test2,$(OBJ)) $(patsubst %.c,%-enzyme-test3,$(OBJ))
+
+generate-testfiles:
+	rm testfiles/*
+	python gentests.py $(patsubst %.c,%-enzyme0,$(OBJ))
+	python gentests.py $(patsubst %.c,%-enzyme1,$(OBJ))
+	python gentests.py $(patsubst %.c,%-enzyme2,$(OBJ))
+	python gentests.py $(patsubst %.c,%-enzyme3,$(OBJ))
+
+clean-%:
+	rm -f build/%*
+
+clean:
+	rm -f build/*
+
+

enzyme/functional_tests_c/gentests.py

@@ -0,0 +1,10 @@
+import sys
+
+for i in range(1,len(sys.argv)):
+  content = open('test.template').read()
+  content = content.replace("@NAME@", sys.argv[i])
+  if sys.argv[i].startswith('FAIL_'):
+    content = content.replace("@EXPECTFAIL@", "; XFAIL: *")
+  else:
+    content = content.replace("@EXPECTFAIL@", "")
+  open('./testfiles/'+sys.argv[i]+".test", 'w+').write(content)

enzyme/functional_tests_c/insertsort_sum.c

@@ -0,0 +1,103 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <math.h>
+#include <assert.h>
+
+#define __builtin_autodiff __enzyme_autodiff
+
+double __enzyme_autodiff(void*, ...);
+
+// size of array
+float* unsorted_array_init(int N) {
+  float* arr = (float*) malloc(sizeof(float) * N);
+  for (int i = 0; i < N; i++) {
+    arr[i] = 1.0*(i%2);
+  }
+  return arr;
+}
+
+// sums the first half of a sorted array.
+void insertsort_sum (float* array, int N, float* ret) {
+  float sum = 0;
+  //qsort(array, N, sizeof(float), cmp);
+
+  for (int i = 1; i < N; i++) {
+    int j = i;
+    while (j > 0 && array[j-1] < array[j]) {
+      float tmp = array[j];
+      array[j] = array[j-1];
+      array[j-1] = tmp;
+      j -= 1;
+    }
+  }
+
+
+  for (int i = 0; i < N/2; i++) {
+    printf("Val: %f\n", array[i]);
+    sum += array[i];
+  }
+  *ret = sum;
+}
+
+
+
+
+int main(int argc, char** argv) {
+
+
+
+  float a = 2.0;
+  float b = 3.0;
+
+
+
+  float da = 0;
+  float db = 0;
+
+
+  float ret = 0;
+  float dret = 1.0;
+
+  int N = 10;
+  int dN = 0;
+  float* array = unsorted_array_init(N);
+  float* d_array = (float*) malloc(sizeof(float)*N);
+  for (int i = 0; i < N; i++) {
+    d_array[i] = 0.0;
+  }
+
+  printf("Array before sorting:\n");
+  for (int i = 0; i < N; i++) {
+    printf("%d:%f\n", i, array[i]); 
+  }
+
+  //insertsort_sum(array, N, &ret);
+
+  printf("Array after sorting:\n");
+  for (int i = 0; i < N; i++) {
+    printf("%d:%f\n", i, array[i]); 
+  }
+
+
+  printf("The total sum is %f\n", ret);
+
+  __builtin_autodiff(insertsort_sum, array, d_array, N, &ret, &dret);
+
+  for (int i = 0; i < N; i++) {
+    printf("Diffe for index %d is %f\n", i, d_array[i]);
+    if (i%2 == 0) {
+      assert(d_array[i] == 0.0);
+    } else {
+      assert(d_array[i] == 1.0);
+    }
+  }
+
+  //__builtin_autodiff(compute_loops, &a, &da, &b, &db, &ret, &dret);
+
+
+  //assert(da == 100*1.0f);
+  //assert(db == 100*1.0f);
+
+  //printf("hello! %f, res2 %f, da: %f, db: %f\n", ret, ret, da,db);
+  return 0;
+}