User:Kr/A Thread's Life

From Apache OpenOffice Wiki
Jump to: navigation, search
The printable version is no longer supported and may have rendering errors. Please update your browser bookmarks and please use the default browser print function instead.

Type: Proposal Status: Draft

Todos:

  • Rework and aligned examples.
  • Convert SAL examples to 'C'
  • Bug examples for SAL threads.
  • Issues for the bugs.

Two types of Threads

Activity

  • Activities keep the process alive.
  • The last activity terminating leads to the termination of the entire process.
  • An activity must be detached, as it is not going to be joined.

Examples

  • A thread dispatching user interface events.
  • A thread dispatching remote UNO messages.

Daemon

  • Daemons don't keep the process alive.
  • A daemon must be joinable, as it is going to be joined during shutdown of the subsystem.
  • The termination of a daemon may be enforced by asking it for cancellation (pthread_cancel) or by killing (pthread_kill) it.

Sleepy Daemons

Daemons waiting for an event (e.g. a condition) to do something, going to sleep afterwards again.

Short lived Daemons

Daemons doing something asynchronously, terminating afterwards.

Examples

  • A thread flushing a cache every once a while.

Notes

  • Threads may dynamically switch from one type to the other and vice versa.
  • The "main" thread starts an activity.

Example

Single Executable using pthreads

//
// gcc -Wall -g example.c -o main.bin -lpthread
// 

#include <stdlib.h>
#include <stdio.h>
#include <pthread.h>
#include <string.h>
#include <unistd.h>


void my_pthread_cancel(pthread_t thread) {
    int res;
    if ((res = pthread_cancel(thread))) {
        fprintf(stderr, "%s(%p) => %s\n", __PRETTY_FUNCTION__, (void*)thread, strerror(res));
        // There seems to be an issue on Linux with pthread_cancel,
        // returning an error in case "thread" has already terminated, 
        // though not joined.
//        abort();
    }
}

void my_pthread_join(pthread_t th, void **thread_return) {
    int res;
    if ((res = pthread_join(th, thread_return))) {
        fprintf(stderr, "%s - %s\n", __PRETTY_FUNCTION__, strerror(res));
        abort();
    }
}


typedef struct Daemon  Daemon;

struct Daemon {
    Daemon * previous_;
    Daemon * next_;

    void (*func_)(void);
    pthread_t hd_;
    int       finished_;
};

static pthread_mutex_t daemons_mutex_ = PTHREAD_MUTEX_INITIALIZER;
static Daemon        * firstDaemon_   = NULL;
static unsigned    int daemons_       = 0; 


static void daemons_join_(int finished) {
    pthread_mutex_lock(&daemons_mutex_);
    Daemon * daemon = firstDaemon_;
    while (daemon) {
        Daemon * next = daemon->next_;

        if (!finished || daemon->finished_) {
            -- daemons_;
            
            if (daemon->next_)
                daemon->next_->previous_ = daemon->previous_;
            
            if (daemon->previous_)
                daemon->previous_->next_ = daemon->next_;
            
            if (firstDaemon_ == daemon)
                firstDaemon_ = daemon->next_;

            pthread_mutex_unlock(&daemons_mutex_);
            my_pthread_join(daemon->hd_, NULL);
            pthread_mutex_lock(&daemons_mutex_);

            daemon->hd_ = 0xffffffff;

            free(daemon);

            daemon = firstDaemon_;
        }
        else
            daemon = next;
    }
    pthread_mutex_unlock(&daemons_mutex_);
}

/* Daemon finish function. */
static void daemon_finish(Daemon * daemon) {
    int oldstate;
    pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &oldstate);

    daemons_join_(1);

    pthread_mutex_lock(&daemons_mutex_);
    daemon->finished_ = 1;
    pthread_mutex_unlock(&daemons_mutex_);

    pthread_setcancelstate(oldstate, NULL);
}

static void * daemon_func(Daemon * daemon) {
    pthread_cleanup_push((void (*)(void *))daemon_finish, daemon);

    daemon->func_();

    pthread_cleanup_pop(1);

    return NULL;
}

/* Create a "Daemon" struct and register it. 
   Start the "Daemon". */
static void daemon_create_(void (*func)(void)) {
    Daemon * daemon   = malloc(sizeof(Daemon));
    daemon->func_     = func;
    daemon->finished_ = 0;

    pthread_mutex_lock(&daemons_mutex_);

    daemon->previous_ = NULL;
    daemon->next_ = firstDaemon_;

    if (firstDaemon_) 
        firstDaemon_->previous_ = daemon;

    firstDaemon_ = daemon;
    ++ daemons_;

    pthread_attr_t attr;
    pthread_attr_init(&attr);
    pthread_attr_setdetachstate(&attr, 0);
    pthread_create(&daemon->hd_, NULL, (void *(*)(void *))daemon_func, daemon);
    pthread_attr_destroy(&attr);

    pthread_mutex_unlock(&daemons_mutex_);
}

/* Ask all registered daemons for cancellation and join them. */
static void daemons_cancelAndJoin_(void) {
    fprintf(stderr, "Joining daemons ... ");

    pthread_mutex_lock(&daemons_mutex_);
    Daemon * daemon = firstDaemon_;
    while (daemon) {
        my_pthread_cancel(daemon->hd_);
        // respectively
        //pthread_kill();

        fprintf(stderr, "# ");

        daemon = daemon->next_;
    }
    pthread_mutex_unlock(&daemons_mutex_);

    daemons_join_(0);
    fprintf(stderr, "done\n");
}


static unsigned    int activities_     = 1; // The "main" thread is the first "activity".
static pthread_mutex_t activity_mutex_ = PTHREAD_MUTEX_INITIALIZER;

static void activity_terminate_(void) {
    int use_exit = 0;
    pthread_mutex_lock(&activity_mutex_);
    -- activities_;
    use_exit = activities_ == 0;
    pthread_mutex_unlock(&activity_mutex_);

    // Terminate the process, if this was the last "activity"!
    if (use_exit) 
        exit(0);

    else
        pthread_exit(NULL);
}

static void * activity_func_(void (*func)(void)) {
    func();

    activity_terminate_();

    return NULL;
}

static void activity_create_(void (*func)(void)) {
    pthread_attr_t attr;
    
    /* Activities need to be detached. */
    pthread_attr_init(&attr);
    pthread_attr_setdetachstate(&attr, 1);
    
    pthread_t bt;
    pthread_create(&bt, &attr, (void *(*)(void *))activity_func_, func);
    pthread_attr_destroy(&attr);
    
    pthread_mutex_lock(&activity_mutex_);
    ++ activities_;
    pthread_mutex_unlock(&activity_mutex_);
}


/* Some daemon functions. */
static void log_threads(void) {
    fprintf(stderr, "activities: %u  daemons: %u\n", activities_, daemons_);
}

static void log_daemon_(void) {
    while (1) {
        log_threads();

        sleep(1); /* This is a cancellation point. */
    }
}

static void some_daemon_(void) {
    int n = 10;
    while (n) {
        sleep(1);
        
        if (rand() < (RAND_MAX / (daemons_ + 1)))
            daemon_create_(some_daemon_);

        -- n;
    }
}

/* Some activity functions. */
static void some_activity_(void) {
    int n = 3;
    while (n) {
        sleep(1);
        
        if (rand() < (RAND_MAX / (activities_ + 1)))
            activity_create_(some_activity_);

        if (rand() < (RAND_MAX / (daemons_ + 1)))
            daemon_create_(some_daemon_);

        -- n;
    }
}

static void main_activity_(void) {
    srand (time (0));

    daemon_create_(log_daemon_);
    daemon_create_(some_daemon_);
    activity_create_(some_activity_);

    sleep(3);
    daemon_create_(some_daemon_);

    sleep(5);
}

int main(void) {
    atexit(log_threads);
    atexit(daemons_cancelAndJoin_);

    main_activity_();

    activity_terminate_();

    return 0;
}

Executable and Library using pthreads

The Main Program

/*
** gcc -Wall -g main.c -o main.bin -lpthread -ldl
*/ 
#include "library.h"

#include <stdlib.h>
#include <stdio.h>
#include <pthread.h>
#include <string.h>
#include <unistd.h>
#include <dlfcn.h>


static unsigned    int activities_     = 1; /* The "main" thread is the first "activity". */
static pthread_mutex_t activity_mutex_ = PTHREAD_MUTEX_INITIALIZER;

static void activity_terminate_(void) {
    int use_exit = 0;
    pthread_mutex_lock(&activity_mutex_);
    -- activities_;
    use_exit = activities_ == 0;
    pthread_mutex_unlock(&activity_mutex_);

    /* Terminate the process, if this was the last "activity"! */
    if (use_exit)
        exit(0);

    else
        pthread_exit(NULL);
}

static void * activity_func_(void (*func)(void)) {
    func();

    activity_terminate_();

    return NULL;
}

static void activity_create_(void (*func)(void)) {
    pthread_attr_t attr;
    
    /* Activities need to be detached. */
    pthread_attr_init(&attr);
    pthread_attr_setdetachstate(&attr, 1);
    
    pthread_t bt;
    pthread_create(&bt, &attr, (void *(*)(void *))activity_func_, func);
    pthread_attr_destroy(&attr);
    
    pthread_mutex_lock(&activity_mutex_);
    ++ activities_;
    pthread_mutex_unlock(&activity_mutex_);
}


/* Some daemon functions. */
static daemon_create_t    * pdaemon_create_;
static daemons_getCount_t * pdaemons_getCount_;


static void log_threads(void) {
    fprintf(stderr, "activities: %u  daemons: %u\n", activities_, pdaemons_getCount_ ? pdaemons_getCount_() : 0);
}

static void log_daemon_(void) {
    while (1) {
        log_threads();

        /* This is a cancellation point. */
        sleep(1); 
    }
}

static void some_daemon_(void) {
    int n = 10;
    while (n) {
        sleep(1);
        
        if (rand() < (RAND_MAX / (pdaemons_getCount_() + 1)))
            pdaemon_create_(some_daemon_);

        -- n;
    }
}

/* Some activity functions. */
static void some_activity_(void) {
    int n = 3;
    while (n) {
        sleep(1);
        
        if (rand() < (RAND_MAX / (activities_ + 1)))
            activity_create_(some_activity_);

        if (rand() < (RAND_MAX / (pdaemons_getCount_() + 1)))
            pdaemon_create_(some_daemon_);

        -- n;
    }
}

static void main_activity_(void) {
    srand(time (0));

    pdaemon_create_(log_daemon_);
    pdaemon_create_(some_daemon_);
    activity_create_(some_activity_);
    
    sleep(3);
    pdaemon_create_(some_daemon_);

    sleep(5);
}

static void * dlhd_ = NULL;

static void close_library(void) {
    daemons_cancelAndJoin_t * pdaemons_cancelAndJoin = dlsym(dlhd_, "daemons_cancelAndJoin");
    if (pdaemons_cancelAndJoin) pdaemons_cancelAndJoin();
    dlclose(dlhd_);
    pdaemon_create_ = NULL;
    pdaemons_getCount_ = NULL;
}

int main(void) {
    atexit(log_threads);
    atexit(close_library);

    dlhd_ = dlopen("library.so", RTLD_NOW);
    pdaemon_create_    = dlsym(dlhd_, "daemon_create");
    pdaemons_getCount_ = dlsym(dlhd_, "daemons_getCount");

    main_activity_();

    activity_terminate_();

    return 0;
}
</code>

====Library Header====
<code>[cpp]
#ifndef library_h_included
#  define library_h_included


typedef void daemon_create_t        (void (*func)(void));
typedef int  daemons_getCount_t     (void);
typedef void daemons_cancelAndJoin_t(void);


void daemon_create        (void (*func)(void));
int  daemons_getCount     (void);
void daemons_cancelAndJoin(void);


#endif
</code>

====Library Implementation====
<code>[cpp]
/*
** gcc -Wall -g library.c -shared -o library.so -lpthread
*/ 
#include "library.h"

#include <stdlib.h>
#include <stdio.h>
#include <pthread.h>
#include <string.h>
#include <unistd.h>


static void my_pthread_cancel(pthread_t thread) {
    int res;
    if ((res = pthread_cancel(thread))) {
        fprintf(stderr, "%s(%p) => %s\n", __PRETTY_FUNCTION__, (void*)thread, strerror(res));
        /* There seems to be an issue on Linux with pthread_cancel,
        ** returning an error in case "thread" has already terminated, 
        ** though not joined. */
/*         abort(); */
    }
}

static void my_pthread_join(pthread_t th, void **thread_return) {
    int res;
    if ((res = pthread_join(th, thread_return))) {
        fprintf(stderr, "%s - %s\n", __PRETTY_FUNCTION__, strerror(res));
        abort();
    }
}


typedef struct Daemon  Daemon;

struct Daemon {
    Daemon * previous_;
    Daemon * next_;

    void (*func_)(void);
    pthread_t hd_;
    int       finished_;
};

static pthread_mutex_t daemons_mutex_ = PTHREAD_MUTEX_INITIALIZER;
static Daemon        * firstDaemon_   = NULL;
static unsigned    int daemons_       = 0; 

int daemons_getCount(void) {
    return daemons_;
}

/* Joins all or only "finished" daemons. */
static void daemons_join_(int finished) {
    pthread_mutex_lock(&daemons_mutex_);
    Daemon * daemon = firstDaemon_;
    while (daemon) {
        Daemon * next = daemon->next_;

        if (!finished || daemon->finished_) {
            -- daemons_;
            
            if (daemon->next_)
                daemon->next_->previous_ = daemon->previous_;
            
            if (daemon->previous_)
                daemon->previous_->next_ = daemon->next_;
            
            if (firstDaemon_ == daemon)
                firstDaemon_ = daemon->next_;

            pthread_mutex_unlock(&daemons_mutex_);
            my_pthread_join(daemon->hd_, NULL);
            pthread_mutex_lock(&daemons_mutex_);

            daemon->hd_ = 0xffffffff;

            free(daemon);

            daemon = firstDaemon_;
        }
        else
            daemon = next;
    }
    pthread_mutex_unlock(&daemons_mutex_);
}

/* Daemon finish function. */
static void daemon_finish_(Daemon * daemon) {
    int oldstate;
    pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &oldstate);

    daemons_join_(1);

    pthread_mutex_lock(&daemons_mutex_);
    daemon->finished_ = 1;
    pthread_mutex_unlock(&daemons_mutex_);

    pthread_setcancelstate(oldstate, NULL);
}

static void * daemon_func_(Daemon * daemon) {
    pthread_cleanup_push((void (*)(void *))daemon_finish_, daemon);
    daemon->func_();
    pthread_cleanup_pop(1);

    return NULL;
}

/* Create a "Daemon" struct and register it. Start the "Daemon". */
void daemon_create(void (*func)(void)) {
    Daemon * daemon   = malloc(sizeof(Daemon));
    daemon->func_     = func;
    daemon->finished_ = 0;

    pthread_mutex_lock(&daemons_mutex_);

    daemon->previous_ = NULL;
    daemon->next_ = firstDaemon_;

    if (firstDaemon_) 
        firstDaemon_->previous_ = daemon;

    firstDaemon_ = daemon;
    ++ daemons_;

    pthread_attr_t attr;
    pthread_attr_init(&attr);
    pthread_attr_setdetachstate(&attr, 0);
    pthread_create(&daemon->hd_, NULL, (void *(*)(void *))daemon_func_, daemon);
    pthread_attr_destroy(&attr);

    pthread_mutex_unlock(&daemons_mutex_);
}

/* Ask all registered daemons for cancellation and join them. */
/* Calling pthread_canel from within _fini leads to deadlocks.*/
/* static void __attribute__((destructor)) daemons_cancelAndJoin_(void) {*/

void daemons_cancelAndJoin(void) {
    fprintf(stderr, "Cancelling daemons ... ");

    pthread_mutex_lock(&daemons_mutex_);
    Daemon * daemon = firstDaemon_;
    while (daemon) {
        my_pthread_cancel(daemon->hd_);
        /* respectively */
        /*pthread_kill();*/

        fprintf(stderr, "# ");

        daemon = daemon->next_;
    }
    pthread_mutex_unlock(&daemons_mutex_);
    fprintf(stderr, "done\n");


    fprintf(stderr, "Joining daemons ... ");
    daemons_join_(0);
    fprintf(stderr, "done\n");
}

Executable and Library using SAL

SAL Thread Issues:

  • Different semantic of UNIX and Win32 implementations
  • Problematic "pthread_detach" in UNIX usage, likely leading to races if trying to join a thread.
  • Problematic "CloseHandle" in Win32 usage, only easily possible to create "detached" (daemon) threads if calling "osl_destroyThread" from within the new thread.

RFE:

  • Reliable thread cancellation mechanism ...
  • Result setting function. Needed in multi-thread keeping process alive scenario.
  • Passing of the "oslThread" object to the worker function, otherwise it must be explicitly passed with help of conditions etc.
  • Better named "osl_scheduleThread".
  • A way to get this threads "oslThread".
  • Missing pendant to "pthread_exit" / "ExitThread" and therefore also missing clean-up handlers etc.
  • Less capable cancellation mechanism compared to "pthread_cancel"

How-to Build

Get an OOo build environment
mkdir salexample
mkdir salexample/source
mkdir salexample/prj
touch salexample/prj/d.lst
cd salexample/source
dmake

The Program

#include "library.hxx"

#include <sal/main.h>
#include <osl/interlck.h>
#include <osl/thread.h>
#include <osl/module.hxx>
#include <rtl/alloc.h>

#include <stdio.h>
#include <time.h>

#ifdef WNT
extern "C" void _endthread(void);

#else
# include <pthread.h>

#endif


static oslInterlockedCount activities_ = 1; /* The "main" thread is the first "activity". */

static void activity_terminate_(void) {
    int use_exit = 0;
    
    use_exit = osl_decrementInterlockedCount(&activities_) == 0;

    /* Terminate the process, if this was the last "activity"! */
    if (use_exit)
        exit(0);

    else
#ifdef WNT
        _endthread();
#else
        pthread_exit(NULL);
#endif
}

typedef void activity_worker_t(void);

static void activity_func_(activity_worker_t ** ffunc) {
    activity_worker_t * func = *ffunc;
    rtl_freeMemory(ffunc);

    func();

    activity_terminate_();
}

static void activity_create_(activity_worker_t * func) {
    activity_worker_t ** ffunc = (activity_worker_t **)rtl_allocateMemory(sizeof(activity_worker_t *));
    *ffunc = func;
    oslThread thread = osl_createThread((oslWorkerFunction)activity_func_, ffunc);

    /* Activities need to be detached. */
#ifndef UNX
    TimeValue oneSecond_ = {1, 0};
    osl_waitThread(&oneSecond_); 
    
    osl_destroyThread(thread);

#else
    (void)thread;

#endif

    osl_incrementInterlockedCount(&activities_);
}


/* Some daemon functions. */
static daemon_create_t    * pdaemon_create_;
static daemons_getCount_t * pdaemons_getCount_;


extern "C" void log_threads(void) {
    fprintf(stderr, "activities: %u  daemons: %u\n", activities_, pdaemons_getCount_ ? pdaemons_getCount_() : 0);
}

static TimeValue oneSecond_ = {1, 0};

static void log_daemon_(oslThread thread) {
    fprintf(stderr, "\tlog daemon starting ...\n");

    while (osl_scheduleThread(thread)) {
        log_threads();

        osl_waitThread(&oneSecond_); 
    }

    fprintf(stderr, "\tlog daemon finishing ...\n");
}

static void some_daemon_(oslThread thread) /*throw (int)*/ {
    fprintf(stderr, "\tsome daemon starting ...\n");

    int n = 10;
    while (osl_scheduleThread(thread) && n > 0) {
        osl_waitThread(&oneSecond_); 

        if (rand() < (RAND_MAX / (pdaemons_getCount_() + 1)))
            pdaemon_create_(some_daemon_, thread);

        -- n;
    }
    fprintf(stderr, "\tsome daemon finishing ...\n");
}

/* Some activity functions. */
static void some_activity_(void) {
    fprintf(stderr, "\tsome activitiy starting ...\n");

    srand(time (0));

    int n = 3;
    while (n) {
        osl_waitThread(&oneSecond_); 
        
        if (rand() < (RAND_MAX / (activities_ + 1)))
            activity_create_(some_activity_);

        if (rand() < (RAND_MAX / (pdaemons_getCount_() + 1)))
            pdaemon_create_(some_daemon_, NULL);

        -- n;
    }

    fprintf(stderr, "\tsome activitiy dying ...\n");
}

static osl::Module library;

#if defined GCC
# define DAEMON_CREATE          "_Z13daemon_createPFvPvES_"
# define DAEMONS_GETCOUNT       "_Z16daemons_getCountv"
# define DAEMONS_CANCELANDJOIN  "_Z21daemons_cancelAndJoinv"

#elif defined __SUNPRO_CC
# define DAEMON_CREATE          "__1cNdaemon_create6FpFpv_v0_v_"
# define DAEMONS_GETCOUNT       "__1cQdaemons_getCount6F_i_"
# define DAEMONS_CANCELANDJOIN  "__1cVdaemons_cancelAndJoin6F_v_"

#elif defined _MSC_VER
# define DAEMON_CREATE          "?daemon_create@@YAXP6AXPAX@Z0@Z"
# define DAEMONS_GETCOUNT       "?daemons_getCount@@YAHXZ"
# define DAEMONS_CANCELANDJOIN  "?daemons_cancelAndJoin@@YAXXZ"

#else
# error "unknown compiler"

#endif

extern "C" void close_library(void) {
    daemons_cancelAndJoin_t * pdaemons_cancelAndJoin = 
         (daemons_cancelAndJoin_t *)library.getFunctionSymbol(rtl::OUString(RTL_CONSTASCII_USTRINGPARAM(DAEMONS_CANCELANDJOIN)));
    pdaemons_cancelAndJoin();
    library.unload();
    pdaemon_create_ = NULL;
    pdaemons_getCount_ = NULL;
}

SAL_IMPLEMENT_MAIN()
{
    atexit(log_threads);
    atexit(close_library);

    library.load(rtl::OUString(RTL_CONSTASCII_USTRINGPARAM("library" SAL_DLLEXTENSION)));
    pdaemon_create_    = (daemon_create_t *)library.getFunctionSymbol(
        rtl::OUString(RTL_CONSTASCII_USTRINGPARAM(DAEMON_CREATE)));
    pdaemons_getCount_ = (daemons_getCount_t *)library.getFunctionSymbol(
        rtl::OUString(RTL_CONSTASCII_USTRINGPARAM(DAEMONS_GETCOUNT)));

    pdaemon_create_(log_daemon_, NULL);

    some_activity_();

    activity_terminate_();

    return 0;
}

Library Header

[cpp]
#ifndef library_hxx_included
#  define library_hxx_included

#include <osl/thread.h>

#ifdef LIBRARY_IMPL
# define LIBRARY_EXPORT SAL_DLLPUBLIC_EXPORT

#else
# define LIBRARY_EXPORT SAL_DLLPUBLIC_IMPORT

#endif


typedef void daemon_worker_t(oslThread thread);

typedef void daemon_create_t        (daemon_worker_t * func, oslThread thread);
typedef int  daemons_getCount_t     (void);
typedef void daemons_cancelAndJoin_t(void);

LIBRARY_EXPORT void daemon_create        (daemon_worker_t * func, oslThread thread) /*throw (int)*/;
LIBRARY_EXPORT int  daemons_getCount     (void);
LIBRARY_EXPORT void daemons_cancelAndJoin(void);


#endif

Library Implementation

[cpp]
#define LIBRARY_IMPL

#include "library.hxx"

#include <rtl/alloc.h>
#include <osl/mutex.h>
#include <osl/thread.h>

#include <stdio.h>


typedef struct Daemon  Daemon;

struct Daemon {
    Daemon * previous_;
    Daemon * next_;

    daemon_worker_t * func_;
    oslThread         hd_;
    int               finished_;
};

static oslMutex daemons_mutex_;
struct DaemonConstruct {
    DaemonConstruct() {
        daemons_mutex_ = osl_createMutex();
    }

    ~DaemonConstruct() {
        osl_destroyMutex(daemons_mutex_);
    }        
};

static DaemonConstruct daemonConstruct;

static Daemon     * firstDaemon_ = NULL;
static unsigned int daemons_     = 0; 

int daemons_getCount(void) {
    return daemons_;
}

/* Joins all or only "finished" daemons. */
static void daemons_join_(int finished) {
    osl_acquireMutex(daemons_mutex_);
    Daemon * daemon = firstDaemon_;
    while (daemon) {
        Daemon * next = daemon->next_;

        if (!finished || daemon->finished_) {
            -- daemons_;
            
            if (daemon->next_)
                daemon->next_->previous_ = daemon->previous_;
            
            if (daemon->previous_)
                daemon->previous_->next_ = daemon->next_;
            
            if (firstDaemon_ == daemon)
                firstDaemon_ = daemon->next_;

            osl_releaseMutex(daemons_mutex_);
            osl_joinWithThread(daemon->hd_);
            osl_acquireMutex(daemons_mutex_);

            osl_destroyThread(daemon->hd_);

            rtl_freeMemory(daemon);

            daemon = firstDaemon_;
        }
        else
            daemon = next;
    }
    osl_releaseMutex(daemons_mutex_);
}

/* Daemon finish function. */
static void daemon_finish_(Daemon * daemon) {
    daemons_join_(1);

    osl_acquireMutex(daemons_mutex_);
    daemon->finished_ = 1;
    osl_releaseMutex(daemons_mutex_);
}

static void daemon_func_(Daemon * daemon) {
    try {
        daemon->func_(daemon->hd_);
    }
    catch (...) {
        fprintf(stderr, " caught something ...\n");
    }

    daemon_finish_(daemon);
}

/* Create a "Daemon" struct and register it. Start the "Daemon". */
LIBRARY_EXPORT void daemon_create(daemon_worker_t * func, oslThread thread) /*throw (int)*/ {
    Daemon * daemon   = (Daemon *)rtl_allocateMemory(sizeof(Daemon));
    daemon->func_     = func;
    daemon->finished_ = 0;

    osl_acquireMutex(daemons_mutex_);
    // add a cancelation point
    if (thread && !osl_scheduleThread(thread)) {
        osl_releaseMutex(daemons_mutex_);
        rtl_freeMemory(daemon);
        throw int();
    }

    daemon->previous_ = NULL;
    daemon->next_ = firstDaemon_;
    
    if (firstDaemon_) 
        firstDaemon_->previous_ = daemon;
    
    firstDaemon_ = daemon;
    ++ daemons_;
        
    daemon->hd_ = osl_createSuspendedThread((oslWorkerFunction)daemon_func_, daemon);
    osl_resumeThread(daemon->hd_);

    osl_releaseMutex(daemons_mutex_);
}

/* Ask all registered daemons for cancellation and join them. */
/* Calling pthread_canel from within _fini leads to deadlocks.*/
/* static void __attribute__((destructor)) daemons_cancelAndJoin_(void) {*/

void daemons_cancelAndJoin(void) {
    fprintf(stderr, "Cancelling daemons ... ");

    osl_acquireMutex(daemons_mutex_);
    Daemon * daemon = firstDaemon_;
    while (daemon) {
        osl_terminateThread(daemon->hd_);
        /* respectively "osl_killThread" :-) */

        fprintf(stderr, "# ");

        daemon = daemon->next_;
    }
    osl_releaseMutex(daemons_mutex_);
    fprintf(stderr, "done\n");


    fprintf(stderr, "Joining daemons ... ");
    daemons_join_(0);
    fprintf(stderr, "done\n");
}

struct LibExit {
    LibExit() {
        fprintf(stderr, "initializing library ...\n");
    }

    ~LibExit() {
        fprintf(stderr, "de-initializing library ...\n");
//        daemons_cancelAndJoin();
    }
};

static LibExit libExit;

Makefile

PRJ     := ..
PRJNAME := SALTHREADS
TARGET  := main
USE_DEFFILE        := TRUE

ENABLE_EXCEPTIONS  := TRUE

.INCLUDE :  settings.mk


.IF "$(OS)"=="SOLARIS"
DIRECT :=
.ENDIF

APP1TARGET  := main
APP1OBJS    := $(OBJ)$/main.obj
.IF "$(OS)"=="SOLARIS"
APP1STDLIBS := -lrary $(SALLIB) 
.ELSE
APP1STDLIBS := $(SALLIB) 
.ENDIF

SHL1DLLPRE  := 
SHL1TARGET  := library
SHL1OBJS    := $(SLO)$/library.obj
SHL1STDLIBS := $(SALHELPERLIB) $(SALLIB)
#SHL1IMPLIB  := i$(SHL1TARGET)
SHL1DEF     := empty.def


.INCLUDE :  target.mk

The DEF File

HEAPSIZE	  0
EXPORTS
Personal tools