/* -*- c-file-style: "GNU" -*- */
/*
 * Copyright (C) CNRS, INRIA, Universite Bordeaux 1, Telecom SudParis
 * See COPYING in top-level directory.
 */

#ifndef _REENTRANT
#define _REENTRANT
#endif

#include "pomp-lib-dummy/pomp2_lib.h"

#include <dlfcn.h>
#include <eztrace-core/eztrace_config.h>
#include <eztrace-core/eztrace_htable.h>
#include <eztrace-instrumentation/pptrace.h>
#include <eztrace-lib/eztrace.h>
#include <eztrace-lib/eztrace_module.h>
#include <omp.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>

#define CURRENT_MODULE openmp
DECLARE_CURRENT_MODULE;

static int _openmp_initialized = 0;

/* if found, the application has been instrumented with opari, so we can record detail events
 * such as parallel for loops, etc.
 * if not found, we only instrument calls to the GNU OpenMP API (eg. parallel section)
 */
static int pomp2_found = 0;

//#define GOMP_RECORD if (!pomp2_found)

// todo: add hooks for
// OMP_Barrier
// OMP_critical ?

int openmp_parallel_id = -1;
int openmp_for_id = -1;
int openmp_section_id = -1;
int openmp_workshare_id = -1;
int openmp_ordered_id = -1;

int openmp_implicit_barrier_id = -1;
int openmp_barrier_id = -1;

int openmp_critical_id = -1;
int openmp_task_create_id = -1;
int openmp_task_run_id = -1;
int openmp_untied_task_create_id = -1;
int openmp_untied_task_run_id = -1;
int openmp_taskwait_id = -1;

int openmp_setlock_id = -1;
int openmp_unsetlock_id = -1;
int openmp_testlock_id = -1;

int openmp_setnestlock_id = -1;
int openmp_unsetnestlock_id = -1;
int openmp_testnestlock_id = -1;

static void openmp_register_functions();

struct ezt_hashtable lock_map;

typedef int EZT_Region_handle;
typedef struct {
  int creating_thread; // the thread who created the task
  int generation_number; // thread-private generation number of task's creating thread
} EZT_Task_handle;

/* TODO: does it work with nested parallelism ? */
/* each thread has its own counter */
static __thread EZT_Region_handle _next_section_id = -1;

static EZT_Region_handle _get_next_section_id() {
//  /* make sure that the counters of each thread do not collide */
//  if (_next_section_id < 0) {
//    /* EZT_Region_handle is an int, this will work for:
//     * - ~ 2^11 threads
//     * - ~ 2^20 parallel region created by a thread
//     */
//    _next_section_id = omp_get_thread_num() << 20;
//  }

//  _next_section_id = ezt_otf2_register_thread_team();

  return _next_section_id++;
}

#define OMP_ENTER(event_id) do {					\
    if(event_id<0) {							\
      openmp_register_functions();					\
      eztrace_assert(event_id >= 0);					\
    }									\
    EZTRACE_SHOULD_TRACE(OTF2_EvtWriter_Enter(evt_writer,		\
					      NULL,			\
					      ezt_get_timestamp(),	\
					      event_id));		\
  }while(0)

#define OMP_LEAVE(event_id) do {					\
    if(event_id<0) {							\
      openmp_register_functions();					\
      eztrace_assert(event_id >= 0);					\
    }									\
    EZTRACE_SHOULD_TRACE(OTF2_EvtWriter_Leave(evt_writer,		\
					      NULL,			\
					      ezt_get_timestamp(),	\
					      event_id));		\
  }while(0)


/* event issued by the master thread before #pragma omp parallel  */
void openmp_parallel_fork_generic(POMP2_Region_handle* pomp2_handle,
				 int num_threads) {
  EZT_Region_handle *ezt_handle = (EZT_Region_handle*)pomp2_handle;
  *ezt_handle = ezt_otf2_register_thread_team("OpenMP Thread Team", num_threads);
  if(EZTRACE_SAFE) {
    OTF2_ErrorCode err =  OTF2_EvtWriter_ThreadFork( evt_writer,
						     NULL,
						     ezt_get_timestamp(),
						     OTF2_PARADIGM_OPENMP,
						     num_threads);
    eztrace_assert(err == OTF2_SUCCESS);
  }
}

/* event issued by the master thread after a  #pragma omp parallel block */
void openmp_parallel_join_generic() {
  if(EZTRACE_SAFE) {
    OTF2_ErrorCode err =  OTF2_EvtWriter_ThreadJoin( evt_writer,
						     NULL,
						     ezt_get_timestamp(),
						     OTF2_PARADIGM_OPENMP);
    eztrace_assert(err == OTF2_SUCCESS);
  }
}

/* event issued by each thread of an OpenMP team at the beginning of a #pragma omp parallel block */
void openmp_parallel_begin_generic(POMP2_Region_handle* pomp2_handle) {
  if(thread_status == ezt_trace_status_uninitialized) {
    /* this is a newly created thread */
    ezt_init_thread();
  }

  int nb_threads = omp_get_num_threads();
  int my_id = omp_get_thread_num();
  EZT_Region_handle *ezt_handle = (EZT_Region_handle*)pomp2_handle;

  int thread_team = *ezt_handle;
  ezt_otf2_register_thread_team_member(thread_team, my_id, nb_threads);

  if(EZTRACE_SAFE) {

    OTF2_ErrorCode err = OTF2_EvtWriter_ThreadTeamBegin(evt_writer,
							NULL,
							ezt_get_timestamp(),
							thread_team);
    eztrace_assert(err == OTF2_SUCCESS);
    OMP_ENTER(openmp_parallel_id);
  }
}

/* event issued by each thread of an OpenMP team at the end of a #pragma omp parallel block */
void openmp_parallel_end_generic(POMP2_Region_handle* pomp2_handle) {
  EZT_Region_handle *ezt_handle = (EZT_Region_handle*)pomp2_handle;
  int thread_team = *ezt_handle;
  if(EZTRACE_SAFE) {
    OTF2_ErrorCode err = OTF2_EvtWriter_ThreadTeamEnd(evt_writer,
						      NULL,
						      ezt_get_timestamp(),
						      thread_team);
    eztrace_assert(err == OTF2_SUCCESS);
    OMP_LEAVE(openmp_parallel_id);
  }
}

void (*libGOMP_parallel_loop_static_start)(void (*)(void*), void*, unsigned, long, long, long, long);
void (*libGOMP_parallel_loop_runtime_start)(void (*)(void*), void*, unsigned, long, long, long, long);
void (*libGOMP_parallel_loop_dynamic_start)(void (*)(void*), void*, unsigned, long, long, long, long);
void (*libGOMP_parallel_loop_guided_start)(void (*)(void*), void*, unsigned, long, long, long, long);

void (*libGOMP_parallel_start)(void (*fn)(void*), void* data, unsigned num_threads);
void (*libGOMP_parallel_end)();
void (*libGOMP_parallel)(void (*fn)(void*), void* data, unsigned num_threads, unsigned int flags);

void (*libGOMP_critical_start)(void);
void (*libGOMP_critical_end)(void);

/* these function are only available in GOMP 4 */
void (*libGOMP_parallel_loop_static)(void (*)(void*), void*, unsigned, long, long, long, long, unsigned);
void (*libGOMP_parallel_loop_dynamic)(void (*)(void*), void*, unsigned, long, long, long, long, unsigned);
void (*libGOMP_parallel_loop_guided)(void (*)(void*), void*, unsigned, long, long, long, long, unsigned);
void (*libGOMP_parallel_loop_runtime)(void (*)(void*), void*, unsigned, long, long, long, unsigned);

struct gomp_arg_t {
  void (*func)(void*);
  void* data;
  EZT_Region_handle id;
};

/* Function called by GOMP_parallel_start for each thread */
void gomp_new_thread(void* arg) {
  //  FUNCTION_ENTRY;
  struct gomp_arg_t* _arg = (struct gomp_arg_t*)arg;
  void (*func)(void*) = _arg->func;
  void* data = _arg->data;
  EZT_Region_handle section_id = _arg->id;
  POMP2_Region_handle *pomp2_section_id = (POMP2_Region_handle *)&section_id;
  if(!pomp2_found)
    openmp_parallel_begin_generic(pomp2_section_id);
  func(data);
  if(!pomp2_found)
    openmp_parallel_end_generic(pomp2_section_id);
  return;
}

/* generic implementation of parallel loop
 */
#define GOMP_PARALLEL_LOOP_GENERIC(fn, data, varname, gomp_func, section_id)                                   \
  {                                                                                                            \
    EZTRACE_PROTECT_ON();                                                                                      \
    /* Since the runtime functions provide more information, let's use it instead of the compiler functions */ \
    /* call unprotected function 'cause the block is protected */                                              \
    struct gomp_arg_t* varname = (struct gomp_arg_t*)malloc(sizeof(struct gomp_arg_t));                        \
    varname->func = fn;                                                                                        \
    varname->data = data;                                                                                      \
    varname->id = section_id;                                                                                  \
    EZTRACE_PROTECT_OFF();                                                                                     \
    gomp_func;                                                                                                 \
  }

#define GOMP4_PARALLEL_LOOP_GENERIC(fn, data, num_threads, varname, gomp_func) \
  {									\
    EZT_Region_handle section_id = -1;					\
    POMP2_Region_handle *pomp2_section_id = (POMP2_Region_handle *)&section_id;	\
    if(!pomp2_found) {							\
      /* the application was not instrumented, so we need to record events when intercepting the libGOMP API */ \
      openmp_parallel_fork_generic(pomp2_section_id, num_threads);	\
    }									\
    GOMP_PARALLEL_LOOP_GENERIC(fn, data, varname, gomp_func, section_id); \
  }

/* generic implementation of parallel loop
 */

#define GOMP3_PARALLEL_LOOP_GENERIC(fn, data, varname, gomp_func)	\
  {									\
    OMP_ENTER(openmp_parallel_id);					\
    EZT_Region_handle section_id = _get_next_section_id();		\
    GOMP_PARALLEL_LOOP_GENERIC(fn, data, varname, gomp_func, section_id); \
    return;								\
  }

/* should be called when reaching #pragma omp parallel for schedule(static)
 * However, this function doesn't seem to be called. Let's implement it just in case.
 */
void GOMP_parallel_loop_static_start(void (*fn)(void*), void* data,
                                     unsigned num_threads, long a1, long a2,
                                     long a3, long a4) {
  GOMP3_PARALLEL_LOOP_GENERIC(fn,
                              data,
                              arg,
                              libGOMP_parallel_loop_static_start(gomp_new_thread,
                                                                 arg, num_threads,
                                                                 a1, a2, a3, a4));
}

/* Function called when reaching  #pragma omp parallel for schedule(runtime) */
void GOMP_parallel_loop_runtime_start(void (*fn)(void*), void* data,
                                      unsigned num_threads, long a1, long a2,
                                      long a3, long a4) {
  GOMP3_PARALLEL_LOOP_GENERIC(fn,
                              data,
                              arg,
                              libGOMP_parallel_loop_runtime_start(gomp_new_thread,
                                                                  arg, num_threads,
                                                                  a1, a2, a3, a4));
}

/* Function called when reaching  #pragma omp parallel for schedule(dynamic) */
void GOMP_parallel_loop_dynamic_start(void (*fn)(void*), void* data,
                                      unsigned num_threads, long a1, long a2,
                                      long a3, long a4) {
  GOMP3_PARALLEL_LOOP_GENERIC(fn,
                              data,
                              arg,
                              libGOMP_parallel_loop_dynamic_start(gomp_new_thread,
                                                                  arg, num_threads,
                                                                  a1, a2, a3, a4));
}

/* Function called when reaching  #pragma omp parallel for schedule(guided) */
void GOMP_parallel_loop_guided_start(void (*fn)(void*), void* data,
                                     unsigned num_threads, long a1, long a2,
                                     long a3, long a4) {
  GOMP3_PARALLEL_LOOP_GENERIC(fn,
                              data,
                              arg,
                              libGOMP_parallel_loop_guided_start(gomp_new_thread,
                                                                 arg, num_threads,
                                                                 a1, a2, a3, a4));
}

void GOMP_parallel_loop_static(void (*fn)(void*), void* data,
                               unsigned num_threads, long start, long end,
                               long incr, long chunk_size, unsigned flags) {
  GOMP4_PARALLEL_LOOP_GENERIC(fn,
                              data,
			      num_threads,
                              arg,
                              libGOMP_parallel_loop_static(gomp_new_thread,
                                                           arg, num_threads,
                                                           start, end, incr,
                                                           chunk_size, flags));
}

void GOMP_parallel_loop_dynamic(void (*fn)(void*), void* data,
                                unsigned num_threads, long start, long end,
                                long incr, long chunk_size, unsigned flags) {
  GOMP4_PARALLEL_LOOP_GENERIC(fn,
                              data,
			      num_threads,
                              arg,
                              libGOMP_parallel_loop_dynamic(gomp_new_thread,
                                                            arg, num_threads,
                                                            start, end, incr,
                                                            chunk_size, flags));
}

void GOMP_parallel_loop_guided(void (*fn)(void*), void* data,
                               unsigned num_threads, long start, long end,
                               long incr, long chunk_size, unsigned flags) {
  GOMP4_PARALLEL_LOOP_GENERIC(fn,
                              data,
			      num_threads,
                              arg,
                              libGOMP_parallel_loop_guided(gomp_new_thread,
                                                           arg, num_threads,
                                                           start, end, incr,
                                                           chunk_size, flags));
}

void GOMP_parallel_loop_runtime(void (*fn)(void*), void* data,
                                unsigned num_threads, long start, long end,
                                long incr, unsigned flags) {
  GOMP4_PARALLEL_LOOP_GENERIC(fn,
                              data,
			      num_threads,
                              arg,
                              libGOMP_parallel_loop_runtime(gomp_new_thread,
                                                            arg, num_threads,
                                                            start, end, incr,
                                                            flags));
}

// Called by the main thread (ie. only once) during #pragma omp parallel
// (fork)
void GOMP_parallel_start(void (*fn)(void*), void* data, unsigned num_threads) {
  
  GOMP4_PARALLEL_LOOP_GENERIC(fn, data, num_threads, arg,
                              libGOMP_parallel_start(gomp_new_thread,
						     arg,
                                                     num_threads));
}


/* in recent versions of libgomp, both the beginning and the end of a parallel region are
 * processed by GOMP_parallel
 */
void
GOMP_parallel (void (*fn) (void *), void *data, unsigned num_threads,
	       unsigned int flags) {
  GOMP4_PARALLEL_LOOP_GENERIC(fn, data, num_threads, arg,
                              libGOMP_parallel(gomp_new_thread,
					       arg,
					       num_threads,
					       flags));
  
  if(!pomp2_found)
    openmp_parallel_join_generic();
}

// Called at the end of a parallel section (~ join)
void GOMP_parallel_end() {
  /* Since the runtime functions provide more information, let's use it instead of the compiler functions */
  libGOMP_parallel_end();

  if(!pomp2_found)
    openmp_parallel_join_generic();
  OMP_LEAVE(openmp_parallel_id);		\
}

void GOMP_critical_start() {
  OMP_ENTER(openmp_critical_id);
  libGOMP_critical_start();
}

void GOMP_critical_end() {
  libGOMP_critical_end();
  OMP_LEAVE(openmp_critical_id);
}

/* beginning of pomp2 internals */

void POMP2_Finalize() {
}

void POMP2_Init() {
  /* todo: initialize stuff ? */
}

void POMP2_Off() {
  /* todo: stop recording events ? */
}

void POMP2_On() {
  /* todo: initialize stuff ? */
}

void POMP2_Begin(POMP2_Region_handle* pomp2_handle __attribute__((unused))) {
  /** Called at the begin of a user defined POMP2 region.
   @param pomp2_handle  The handle of the started region.
   */
}

void POMP2_End(POMP2_Region_handle* pomp2_handle __attribute__((unused))) {
}

void POMP2_Assign_handle(
    POMP2_Region_handle* pomp2_handle __attribute__((unused)),
    const char ctc_string[] __attribute__((unused))) {
  /** Registers a POMP2 region and returns a region handle.
     @param pomp2_handle  Returns the handle for the newly registered region.
     @param ctc_string   A string containing the region data.
     */
}

/* end of pomp2 internals */

void POMP2_Atomic_enter(
    POMP2_Region_handle* pomp2_handle __attribute__((unused)),
    const char ctc_string[] __attribute__((unused))) {
  /** Called before an atomic statement.
   @param pomp2_handle The handle of the started region.
   @param ctc_string   Initialization string. May be ignored if \<pomp2_handle\> is already initialized.
   */
}

void POMP2_Atomic_exit(
    POMP2_Region_handle* pomp2_handle __attribute__((unused))) {
}

void POMP2_Implicit_barrier_enter(
    POMP2_Region_handle* pomp2_handle __attribute__((unused)),
    POMP2_Task_handle* pomp2_old_task __attribute__((unused))) {
  /** Called before an implicit barrier.

      \e OpenMP \e 3.0: Barriers can be used as scheduling points for
      tasks. When entering a barrier the task id of the currently
      executing task (\e pomp2_current_task) is saved in \e
      pomp2_old_task, which is defined inside the instrumented user
      code.

      @param pomp2_handle   The handle of the started region.
      @param pomp2_old_task Pointer to a "taskprivate" variable where the current task id is stored.
  */
  OMP_ENTER(openmp_implicit_barrier_id);
}

extern void POMP2_Implicit_barrier_exit(
    POMP2_Region_handle* pomp2_handle __attribute__((unused)),
    POMP2_Task_handle pomp2_old_task __attribute__((unused))) {
  OMP_LEAVE(openmp_implicit_barrier_id);
}

void POMP2_Barrier_enter(
    POMP2_Region_handle* pomp2_handle __attribute__((unused)),
    POMP2_Task_handle* pomp2_old_task __attribute__((unused)),
    const char ctc_string[] __attribute__((unused))) {
  OMP_ENTER(openmp_barrier_id);
}

void POMP2_Barrier_exit(
    POMP2_Region_handle* pomp2_handle __attribute__((unused)),
    POMP2_Task_handle pomp2_old_task __attribute__((unused))) {
  OMP_LEAVE(openmp_barrier_id);
}

void POMP2_Flush_enter(
    POMP2_Region_handle* pomp2_handle __attribute__((unused)),
    const char ctc_string[] __attribute__((unused))) {
  //  FUNCTION_ENTRY;
  //EZTRACE_EVENT_PACKED_0(EZTRACE_POMP2_FLUSH_ENTER);
}

void POMP2_Flush_exit(POMP2_Region_handle* pomp2_handle __attribute__((unused))) {
  //  FUNCTION_ENTRY;
  //EZTRACE_EVENT_PACKED_0(EZTRACE_POMP2_FLUSH_EXIT);
}

void POMP2_Critical_begin(
    POMP2_Region_handle* pomp2_handle __attribute__((unused))) {
  //  FUNCTION_ENTRY;
  //EZTRACE_EVENT_PACKED_0(EZTRACE_POMP2_CRITICAL_BEGIN);
}

void POMP2_Critical_end(
    POMP2_Region_handle* pomp2_handle __attribute__((unused))) {
  //  FUNCTION_ENTRY;
  //EZTRACE_EVENT_PACKED_0(EZTRACE_POMP2_CRITICAL_END);
}

void POMP2_Critical_enter(
    POMP2_Region_handle* pomp2_handle __attribute__((unused)),
    const char ctc_string[] __attribute__((unused))) {
  //  FUNCTION_ENTRY;
  //EZTRACE_EVENT_PACKED_0(EZTRACE_POMP2_CRITICAL_ENTER);
}

void POMP2_Critical_exit(
    POMP2_Region_handle* pomp2_handle __attribute__((unused))) {
  //  FUNCTION_ENTRY;
  //EZTRACE_EVENT_PACKED_0(EZTRACE_POMP2_CRITICAL_EXIT);
}

void POMP2_Master_begin(
    POMP2_Region_handle* pomp2_handle __attribute__((unused)),
    const char ctc_string[] __attribute__((unused))) {
  //  FUNCTION_ENTRY;
  //EZTRACE_EVENT_PACKED_0(EZTRACE_POMP2_MASTER_BEGIN);
}

void POMP2_Master_end(POMP2_Region_handle* pomp2_handle __attribute__((unused))) {
  //  FUNCTION_ENTRY;
  //EZTRACE_EVENT_PACKED_0(EZTRACE_POMP2_MASTER_END);
}


/* event issued by the master thread before #pragma omp parallel  */
void POMP2_Parallel_fork(POMP2_Region_handle* pomp2_handle,
			 int if_clause __attribute__((unused)),
			 int num_threads,
			 POMP2_Task_handle* pomp2_old_task __attribute__((unused)),
			 const char ctc_string[] __attribute__((unused))) {
  openmp_parallel_fork_generic(pomp2_handle, num_threads);
}

/* event issued by the master thread after a  #pragma omp parallel block */
void POMP2_Parallel_join(POMP2_Region_handle* pomp2_handle __attribute__((unused)),
			 POMP2_Task_handle pomp2_old_task __attribute__((unused))) {
  openmp_parallel_join_generic();
}

/* event issued by each thread of an OpenMP team at the beginning of a #pragma omp parallel block */
void POMP2_Parallel_begin(POMP2_Region_handle* pomp2_handle) {
  openmp_parallel_begin_generic(pomp2_handle);
}

/* event issued by each thread of an OpenMP team at the end of a #pragma omp parallel block */
void POMP2_Parallel_end(POMP2_Region_handle* pomp2_handle) {
  openmp_parallel_end_generic(pomp2_handle);
}


/* event issued by each thread at the beginnig of a #pragma omp for loop */
void POMP2_For_enter(POMP2_Region_handle* pomp2_handle __attribute__((unused)),
                     const char ctc_string[] __attribute__((unused))) {
  OMP_ENTER(openmp_for_id);
}

void POMP2_For_exit(POMP2_Region_handle* pomp2_handle __attribute__((unused))) {
  OMP_LEAVE(openmp_for_id);
}

void POMP2_Section_begin(
    POMP2_Region_handle* pomp2_handle __attribute__((unused)),
    const char ctc_string[] __attribute__((unused))) {
  }

void POMP2_Section_end(
    POMP2_Region_handle* pomp2_handle __attribute__((unused))) {
}

void POMP2_Sections_enter(
    POMP2_Region_handle* pomp2_handle __attribute__((unused)),
    const char ctc_string[] __attribute__((unused))) {
  OMP_ENTER(openmp_section_id);
}

void POMP2_Sections_exit(
    POMP2_Region_handle* pomp2_handle __attribute__((unused))) {
  OMP_LEAVE(openmp_section_id);
}

void POMP2_Single_begin(
    POMP2_Region_handle* pomp2_handle __attribute__((unused))) {
}

void POMP2_Single_end(POMP2_Region_handle* pomp2_handle __attribute__((unused))) {
}

void POMP2_Single_enter(
    POMP2_Region_handle* pomp2_handle __attribute__((unused)),
    const char ctc_string[] __attribute__((unused))) {
  OMP_ENTER(openmp_section_id);
}

void POMP2_Single_exit(
    POMP2_Region_handle* pomp2_handle __attribute__((unused))) {
  OMP_LEAVE(openmp_section_id);
}

void POMP2_Workshare_enter(
    POMP2_Region_handle* pomp2_handle __attribute__((unused)),
    const char ctc_string[] __attribute__((unused))) {
  OMP_ENTER(openmp_workshare_id);
}

void POMP2_Workshare_exit(
    POMP2_Region_handle* pomp2_handle __attribute__((unused))) {
  OMP_LEAVE(openmp_workshare_id);
}

void POMP2_Ordered_begin(
    POMP2_Region_handle* pomp2_handle __attribute__((unused))) {
  /** Called at the start of an ordered region.
   @param pomp2_handle  The handle of the region.
   */
  //  FUNCTION_ENTRY;
  // todo :   //EZTRACE_EVENT_PACKED_0(EZTRACE_POMP2_ORDERED_BEGIN);
}

void POMP2_Ordered_end(
    POMP2_Region_handle* pomp2_handle __attribute__((unused))) {
  /* todo: implement that */
}

void POMP2_Ordered_enter(
    POMP2_Region_handle* pomp2_handle __attribute__((unused)),
    const char ctc_string[] __attribute__((unused))) {
  OMP_ENTER(openmp_ordered_id);
}

void POMP2_Ordered_exit(
    POMP2_Region_handle* pomp2_handle __attribute__((unused))) {
  OMP_LEAVE(openmp_ordered_id);
}

static _Thread_local EZT_Task_handle task_handle = {.creating_thread = -1,
						      .generation_number = -1};
#define DEFAULT_STACK_SIZE 1024
struct _ezt_task_stack {
  EZT_Task_handle *stack;
  int alloc_size; // number of allocated spots
  int index; // index of the next empty spot in the stack
};
static _Thread_local struct _ezt_task_stack
_task_stack = {
	       .stack = NULL,
	       .alloc_size = 0,
	       .index = 0};

void task_stack_push(EZT_Task_handle *t) {
  while(_task_stack.index >= _task_stack.alloc_size) {
    /* stack is full. Allocate more spots */
    int new_size = _task_stack.alloc_size ? _task_stack.alloc_size * 2 : DEFAULT_STACK_SIZE;
    void* ptr = realloc(_task_stack.stack, new_size * sizeof(EZT_Task_handle));
    if(!ptr) {
      eztrace_error("Cannot allocated memory\n");
    }
    _task_stack.stack = ptr;
    _task_stack.alloc_size = new_size;
  }

  memcpy(&_task_stack.stack[_task_stack.index], t, sizeof(EZT_Task_handle));
  _task_stack.index++;
}

void task_stack_pop() {
  eztrace_assert(_task_stack.index > 0);
  //#if DEBUG
#if 1
  memset(&_task_stack.stack[_task_stack.index-1], -1, sizeof(EZT_Task_handle));
#endif
  _task_stack.index--;
}

int task_stack_size() { return _task_stack.index; }

EZT_Task_handle * task_stack_top() {
  if(task_stack_size() > 0)
    return &_task_stack.stack[_task_stack.index-1];
  return NULL;
}

void _ezt_task_create(POMP2_Region_handle* pomp2_handle,
		      POMP2_Task_handle* pomp2_new_task,
		      POMP2_Task_handle* pomp2_old_task __attribute__((unused)),
		      int pomp2_if __attribute__((unused)),
		      const char ctc_string[] __attribute__((unused))) {
  /* we could use pomp2_new_task to store the task id, but this would require a mutex
   * (or at least an atomic increment primitive) that would kill the performance.
   * In order to keep the overhead as low as possible, let's only record the event.
   */
  if(EZTRACE_SAFE) {
    EZT_Region_handle *ezt_handle = (EZT_Region_handle*)pomp2_handle;
    int thread_team = *ezt_handle;

    EZT_Task_handle *ezt_new_task = (EZT_Task_handle *) pomp2_new_task;

    if(task_handle.generation_number < 0) {
      task_handle.creating_thread = otf2_thread_id;
      task_handle.generation_number = 0;
    }

    memcpy(ezt_new_task, &task_handle, sizeof(task_handle));
    task_handle.generation_number++;
    OTF2_ErrorCode err = OTF2_EvtWriter_ThreadTaskCreate(evt_writer,
							 NULL,
							 ezt_get_timestamp(),
							 thread_team,
							 otf2_thread_id,
							 ezt_new_task->generation_number);
    eztrace_assert(err == OTF2_SUCCESS);
  }
}

void _ezt_task_begin(POMP2_Region_handle* pomp2_handle,
                      POMP2_Task_handle pomp2_task) {
  EZT_Region_handle *ezt_handle = (EZT_Region_handle*)pomp2_handle;
  int thread_team = *ezt_handle;
  if(EZTRACE_SAFE) {

    EZT_Task_handle *task_handle = (EZT_Task_handle *)&pomp2_task;
    task_stack_push(task_handle);
    OTF2_ErrorCode err = OTF2_EvtWriter_ThreadTaskSwitch(evt_writer,
							 NULL,
							 ezt_get_timestamp(),
							 thread_team,
							 task_handle->creating_thread,
							 task_handle->generation_number);
    eztrace_assert(err == OTF2_SUCCESS);
  }
}
void ezt_task_end(POMP2_Region_handle* pomp2_handle) {

  EZT_Region_handle *ezt_handle = (EZT_Region_handle*)pomp2_handle;
  int thread_team = *ezt_handle;

  if(EZTRACE_SAFE) {
    EZT_Task_handle *current_omp_task = task_stack_top();
    eztrace_assert(current_omp_task);
    eztrace_assert(current_omp_task->creating_thread >= 0);
    eztrace_assert(current_omp_task->generation_number >= 0);

    OTF2_ErrorCode err = OTF2_EvtWriter_ThreadTaskComplete(evt_writer,
							   NULL,
							   ezt_get_timestamp(),
							   thread_team,
							   current_omp_task->creating_thread,
							   current_omp_task->generation_number);
  
    eztrace_assert(err == OTF2_SUCCESS);
#if DEBUG
    current_omp_task->creating_thread = -1;
    current_omp_task->generation_number = -1;
#endif
    task_stack_pop();
  }
}

/** \e OpenMP \e 3.0: When a task encounters a task construct it creates
 a new task. The task may be scheduled for later execution or
 executed immediately. In both cases the pomp-adapter assigns the
 id of the currently active task to \e pomp2_old_task which is
 defined in the instrumented user code.

 @param pomp2_handle   The handle of the region.
 @param pomp2_old_task Pointer to the task id in the instrumented user code
 @param pomp2_if       If an if clause is present on the task
 directive this variable holds the evaluated
 result of the argument of the if
 clause. Else it is 1.
 @param ctc_string     The initialization string.

*/
void POMP2_Task_create_begin(POMP2_Region_handle* pomp2_handle,
			     POMP2_Task_handle* pomp2_new_task,
			     POMP2_Task_handle* pomp2_old_task,
			     int pomp2_if,
			     const char ctc_string[]) {
  _ezt_task_create(pomp2_handle,
		   pomp2_new_task,
		   pomp2_old_task,
		   pomp2_if,
		   ctc_string);
  OMP_ENTER(openmp_task_create_id);
}

void POMP2_Task_create_end(
    POMP2_Region_handle* pomp2_handle __attribute__((unused)),
    POMP2_Task_handle pomp2_old_task __attribute__((unused))) {

  OMP_LEAVE(openmp_task_create_id);
}

void POMP2_Task_begin(POMP2_Region_handle* pomp2_handle,
                      POMP2_Task_handle pomp2_task) {
  OMP_ENTER(openmp_task_run_id);
  _ezt_task_begin(pomp2_handle, pomp2_task);
}

void POMP2_Task_end(POMP2_Region_handle* pomp2_handle __attribute__((unused))) {
  ezt_task_end(pomp2_handle);
  OMP_LEAVE(openmp_task_run_id);
}

void POMP2_Untied_task_create_begin(POMP2_Region_handle* pomp2_handle,
				    POMP2_Task_handle* pomp2_new_task,
				    POMP2_Task_handle* pomp2_old_task __attribute__((unused)),
				    int pomp2_if __attribute__((unused)),
				    const char ctc_string[] __attribute__((unused))) {

  _ezt_task_create(pomp2_handle,
		   pomp2_new_task,
		   pomp2_old_task,
		   pomp2_if,
		   ctc_string);

  OMP_ENTER(openmp_untied_task_create_id);
}

void POMP2_Untied_task_create_end(POMP2_Region_handle* pomp2_handle __attribute__((unused)),
				  POMP2_Task_handle pomp2_old_task __attribute__((unused))) {

  OMP_LEAVE(openmp_untied_task_create_id);
}

void POMP2_Untied_task_begin(POMP2_Region_handle* pomp2_handle,
			     POMP2_Task_handle pomp2_task) {
  
  OMP_ENTER(openmp_untied_task_run_id);
  _ezt_task_begin(pomp2_handle, pomp2_task);
}

void POMP2_Untied_task_end(POMP2_Region_handle* pomp2_handle __attribute__((unused))) {
  ezt_task_end(pomp2_handle);
  OMP_LEAVE(openmp_untied_task_run_id);
}

void POMP2_Taskwait_begin(
    POMP2_Region_handle* pomp2_handle __attribute__((unused)),
    POMP2_Task_handle* pomp2_old_task __attribute__((unused)),
    const char ctc_string[] __attribute__((unused))) {
  OMP_ENTER(openmp_taskwait_id);
}

void POMP2_Taskwait_end( POMP2_Region_handle* pomp2_handle __attribute__((unused)),
			 POMP2_Task_handle pomp2_task __attribute__((unused))) {
  OMP_LEAVE(openmp_taskwait_id);
}

#ifdef OPENMP_FOUND

static _Atomic uint32_t next_lock_id=0;

struct ezt_omp_lock_info {
  uint32_t acquisition_order;
  uint32_t lock_id;
  uint32_t nested_lock_count;
  omp_lock_t* addr;
};

void POMP2_Init_lock(omp_lock_t* s) {
  omp_init_lock(s);
  struct ezt_omp_lock_info* l = malloc(sizeof(struct ezt_omp_lock_info));
  l->acquisition_order = 0;
  l->lock_id = next_lock_id ++;
  l->nested_lock_count = 0;
  l->addr = s;
  ezt_hashtable_insert(&lock_map, hash_function_ptr(s), (void*)l);
  eztrace_assert(ezt_hashtable_get(&lock_map, hash_function_ptr(s)) == l);
}

void POMP2_Destroy_lock(omp_lock_t* s) {
  ezt_hashtable_remove(&lock_map, hash_function_ptr(s));
  omp_destroy_lock(s);
}

void POMP2_Set_lock(omp_lock_t* s) {
  OMP_ENTER(openmp_setlock_id);
  omp_set_lock(s);

  if(EZTRACE_SAFE) {

    struct ezt_omp_lock_info* l = ezt_hashtable_get(&lock_map, hash_function_ptr(s));
    eztrace_assert(l);
    l->nested_lock_count = 1;
    l->acquisition_order++;
    OTF2_ErrorCode err = OTF2_EvtWriter_ThreadAcquireLock(evt_writer,
							  NULL,
							  ezt_get_timestamp(),
							  OTF2_PARADIGM_OPENMP,
							  l->lock_id,
							  l->acquisition_order);
    eztrace_assert(err == OTF2_SUCCESS);
  }
  OMP_LEAVE(openmp_setlock_id);
}

void POMP2_Unset_lock(omp_lock_t* s) {
  OMP_ENTER(openmp_unsetlock_id);

    if(EZTRACE_SAFE) {

      struct ezt_omp_lock_info* l = ezt_hashtable_get(&lock_map, hash_function_ptr(s));
      eztrace_assert(l);
      l->nested_lock_count = 0;

      OTF2_ErrorCode err =  OTF2_EvtWriter_ThreadReleaseLock(evt_writer,
							     NULL,
							     ezt_get_timestamp(),
							     OTF2_PARADIGM_OPENMP,
							     l->lock_id,
							     l->acquisition_order);
      eztrace_assert(err == OTF2_SUCCESS);
    }
    omp_unset_lock(s);
    OMP_LEAVE(openmp_unsetlock_id);
}

int POMP2_Test_lock(omp_lock_t* s) {
  OMP_ENTER(openmp_testlock_id);
  int ret = omp_test_lock(s);
  OMP_LEAVE(openmp_testlock_id);
  return ret;
}

void POMP2_Init_nest_lock(omp_nest_lock_t* s) {
  omp_init_nest_lock(s);

  struct ezt_omp_lock_info* l = malloc(sizeof(struct ezt_omp_lock_info));
  l->acquisition_order = 0;
  l->nested_lock_count = 0;
  l->lock_id = next_lock_id ++;
  l->addr = (omp_lock_t*)s;
  ezt_hashtable_insert(&lock_map, hash_function_ptr(s), (void*)l);
  eztrace_assert(ezt_hashtable_get(&lock_map, hash_function_ptr(s)) == l);
}

void POMP2_Destroy_nest_lock(omp_nest_lock_t* s) {
  ezt_hashtable_remove(&lock_map, hash_function_ptr(s));
  omp_destroy_nest_lock(s);
}

void POMP2_Set_nest_lock(omp_nest_lock_t* s) {
  OMP_ENTER(openmp_setnestlock_id);
  omp_set_nest_lock(s);

  struct ezt_omp_lock_info* l = ezt_hashtable_get(&lock_map, hash_function_ptr(s));
  eztrace_assert(l);

  l->nested_lock_count ++;
  if(l->nested_lock_count==1) {
    l->acquisition_order++;
    if(EZTRACE_SAFE) {
      OTF2_ErrorCode err = OTF2_EvtWriter_ThreadAcquireLock(evt_writer,
							    NULL,
							    ezt_get_timestamp(),
							    OTF2_PARADIGM_OPENMP,
							    l->lock_id,
							    l->acquisition_order);
      eztrace_assert(err == OTF2_SUCCESS);
    }
  }

  OMP_LEAVE(openmp_setnestlock_id);
}

void POMP2_Unset_nest_lock(omp_nest_lock_t* s) {
  OMP_ENTER(openmp_unsetnestlock_id);

  struct ezt_omp_lock_info* l = ezt_hashtable_get(&lock_map, hash_function_ptr(s));
  eztrace_assert(l);
  l->nested_lock_count --;
  if(l->nested_lock_count == 0) {
    if(EZTRACE_SAFE) {
      OTF2_ErrorCode err =  OTF2_EvtWriter_ThreadReleaseLock(evt_writer,
							     NULL,
							     ezt_get_timestamp(),
							     OTF2_PARADIGM_OPENMP,
							     l->lock_id,
							     l->acquisition_order);
      eztrace_assert(err == OTF2_SUCCESS);
    }
  }
  omp_unset_nest_lock(s);
  OMP_LEAVE(openmp_unsetnestlock_id);
}

int POMP2_Test_nest_lock(omp_nest_lock_t* s) {
  OMP_ENTER(openmp_testnestlock_id);
  int ret = omp_test_nest_lock(s);
  OMP_LEAVE(openmp_testnestlock_id);
  return ret;
}

#endif /* OPENMP_FOUND */

void (*libPOMP2_Finalize)();

PPTRACE_START_INTERCEPT_FUNCTIONS(openmp)
INTERCEPT3("GOMP_parallel_start", libGOMP_parallel_start)
INTERCEPT3("GOMP_parallel", libGOMP_parallel)
INTERCEPT3("GOMP_parallel_end", libGOMP_parallel_end)
INTERCEPT3("GOMP_parallel_loop_static_start", libGOMP_parallel_loop_static_start)
INTERCEPT3("GOMP_parallel_loop_runtime_start", libGOMP_parallel_loop_runtime_start)
INTERCEPT3("GOMP_parallel_loop_dynamic_start", libGOMP_parallel_loop_dynamic_start)
INTERCEPT3("GOMP_parallel_loop_guided_start", libGOMP_parallel_loop_guided_start)

INTERCEPT3("GOMP_critical_start", libGOMP_critical_start)
INTERCEPT3("GOMP_critical_end", libGOMP_critical_end)

INTERCEPT3("GOMP_parallel_loop_static", libGOMP_parallel_loop_static)
INTERCEPT3("GOMP_parallel_loop_dynamic", libGOMP_parallel_loop_dynamic)
INTERCEPT3("GOMP_parallel_loop_guided", libGOMP_parallel_loop_guided)
INTERCEPT3("GOMP_parallel_loop_runtime", libGOMP_parallel_loop_runtime)
PPTRACE_END_INTERCEPT_FUNCTIONS(openmp)

static void openmp_register_functions() {
  if(openmp_for_id < 0 ) {
    openmp_for_id = ezt_otf2_register_function("OpenMP For");
    openmp_parallel_id = ezt_otf2_register_function("OpenMP Parallel");
    openmp_section_id = ezt_otf2_register_function("OpenMP Section");
    openmp_workshare_id = ezt_otf2_register_function("OpenMP Workshare");
    openmp_ordered_id = ezt_otf2_register_function("OpenMP Ordered");

    openmp_critical_id = ezt_otf2_register_function("OpenMP critical");
    openmp_implicit_barrier_id = ezt_otf2_register_function("OpenMP implicit barrier");
    openmp_barrier_id = ezt_otf2_register_function("OpenMP barrier");

    openmp_task_create_id = ezt_otf2_register_function("OpenMP Task create");
    openmp_task_run_id = ezt_otf2_register_function("OpenMP Task run");

    openmp_untied_task_create_id = ezt_otf2_register_function("OpenMP Untied Task create");
    openmp_untied_task_run_id = ezt_otf2_register_function("OpenMP Untied  Task run");
    openmp_taskwait_id = ezt_otf2_register_function("OpenMP Task wait");

    openmp_setlock_id = ezt_otf2_register_function("OpenMP Set Lock");
    openmp_unsetlock_id = ezt_otf2_register_function("OpenMP Unset Lock");
    openmp_testlock_id = ezt_otf2_register_function("OpenMP Test Lock");

    openmp_setnestlock_id = ezt_otf2_register_function("OpenMP Set Nest Lock");
    openmp_unsetnestlock_id = ezt_otf2_register_function("OpenMP Unset Nest Lock");
    openmp_testnestlock_id = ezt_otf2_register_function("OpenMP Test Nest Lock");
  }
}

void init_openmp() {
  INSTRUMENT_FUNCTIONS(openmp);

  /* This symbol is only available if the program was compiled with eztrace_cc.
   * Use this information as a test and print a warning message.
   */
  void* has_pomp2_finalize = dlsym(RTLD_NEXT, "POMP2_Finalize");

  if (has_pomp2_finalize) {
    eztrace_log(dbg_lvl_normal, "Intercepting all OpenMP constructs\n");
    pomp2_found = 1;
  } else {
    eztrace_log(dbg_lvl_normal, "Intercepting GNU OpenMP API calls\n");
    eztrace_warn(
        "Only GNU OpenMP runtime functions will be intercepted. For a more precise trace, please instrument your program with eztrace_cc.\n");
    pomp2_found = 0;
  }

  ezt_hashtable_init(&lock_map, 1024);

  openmp_register_functions();

  if (eztrace_autostart_enabled())
    eztrace_start();

  _openmp_initialized = 1;
}

static void finalize_openmp() {
  _openmp_initialized = 0;
  eztrace_stop();
}


void _gomp_init(void) __attribute__((constructor));
void _gomp_init(void) {
  eztrace_log(dbg_lvl_debug, "eztrace_omp constructor starts\n");
  EZT_REGISTER_MODULE(openmp, "Module for OpenMP", init_openmp, finalize_openmp);
  eztrace_log(dbg_lvl_debug, "eztrace_omp constructor ends\n");
}
