taskvine.h

Go to the documentation of this file.

/*
Copyright (C) 2022- The University of Notre Dame
This software is distributed under the GNU General Public License.
See the file COPYING for details.
*/
 
#ifndef TASKVINE_H
#define TASKVINE_H
 
#include <sys/types.h>
#include "timestamp.h"
#include "category.h"
#include "rmsummary.h"
 
struct vine_manager;
struct vine_task;
struct vine_file;
 
#define VINE_DEFAULT_PORT 9123               
#define VINE_RANDOM_PORT  0                  
#define VINE_WAIT_FOREVER -1                 
typedef enum {
        VINE_TRANSFER_ALWAYS = 0, 
        VINE_FIXED_LOCATION  = 1,   
        VINE_WATCH = 2,           
        VINE_FAILURE_ONLY = 4,    
        VINE_SUCCESS_ONLY = 8,    
        VINE_RETRACT_ON_RESET = 16,  
        VINE_MOUNT_SYMLINK = 32,   
        VINE_MOUNT_MKDIR = 64     
} vine_mount_flags_t;
 
typedef enum {
        VINE_CACHE_LEVEL_TASK=0,      
        VINE_CACHE_LEVEL_WORKFLOW=1,  
        VINE_CACHE_LEVEL_WORKER=2,    
        VINE_CACHE_LEVEL_FOREVER=3    
} vine_cache_level_t;
 
 
typedef enum {
        VINE_PEER_NOSHARE = 1,     
        VINE_UNLINK_WHEN_DONE = 2  
} vine_file_flags_t;
 
 
typedef enum {
        VINE_SCHEDULE_UNSET = 0, 
        VINE_SCHEDULE_FCFS,      
        VINE_SCHEDULE_FILES,     
        VINE_SCHEDULE_TIME,      
        VINE_SCHEDULE_RAND,      
        VINE_SCHEDULE_WORST      
} vine_schedule_t;
 
typedef enum {
        VINE_RESULT_SUCCESS             = 0,      
        VINE_RESULT_INPUT_MISSING       = 1,      
        VINE_RESULT_OUTPUT_MISSING      = 2,      
        VINE_RESULT_STDOUT_MISSING      = 4,      
        VINE_RESULT_SIGNAL              = 1 << 3, 
        VINE_RESULT_RESOURCE_EXHAUSTION = 2 << 3, 
        VINE_RESULT_MAX_END_TIME        = 3 << 3, 
        VINE_RESULT_UNKNOWN             = 4 << 3, 
        VINE_RESULT_FORSAKEN            = 5 << 3, 
        VINE_RESULT_MAX_RETRIES         = 6 << 3, 
        VINE_RESULT_MAX_WALL_TIME       = 7 << 3, 
        VINE_RESULT_RMONITOR_ERROR      = 8 << 3, 
        VINE_RESULT_OUTPUT_TRANSFER_ERROR = 9 << 3,  
        VINE_RESULT_FIXED_LOCATION_MISSING = 10 << 3, 
        VINE_RESULT_CANCELLED = 11<<3, 
        VINE_RESULT_LIBRARY_EXIT        = 12 << 3 
} vine_result_t;
 
typedef enum {
        VINE_ALLOCATION_MODE_FIXED = CATEGORY_ALLOCATION_MODE_FIXED,
 
        VINE_ALLOCATION_MODE_MAX = CATEGORY_ALLOCATION_MODE_MAX,
 
        VINE_ALLOCATION_MODE_MIN_WASTE = CATEGORY_ALLOCATION_MODE_MIN_WASTE,
 
        VINE_ALLOCATION_MODE_MAX_THROUGHPUT = CATEGORY_ALLOCATION_MODE_MAX_THROUGHPUT,
 
    VINE_ALLOCATION_MODE_GREEDY_BUCKETING = CATEGORY_ALLOCATION_MODE_GREEDY_BUCKETING,
 
    VINE_ALLOCATION_MODE_EXHAUSTIVE_BUCKETING = CATEGORY_ALLOCATION_MODE_EXHAUSTIVE_BUCKETING
} vine_category_mode_t;
 
struct vine_stats {
        /* Stats for the current state of workers: */
        int workers_connected;    
        int workers_init;         
        int workers_idle;         
        int workers_busy;         
        int workers_able;         
        /* Cumulative stats for workers: */
        int workers_joined;       
        int workers_removed;      
        int workers_released;     
        int workers_idled_out;    
        int workers_slow;         
        int workers_blocked ;     
        int workers_lost;         
        /* Stats for the current state of tasks: */
        int tasks_waiting;        
        int tasks_on_workers;     
        int tasks_running;        
        int tasks_with_results;   
        /* Cumulative stats for tasks: */
        int tasks_submitted;           
        int tasks_dispatched;          
        int tasks_done;                
        int tasks_failed;              
        int tasks_cancelled;           
        int tasks_exhausted_attempts;  
        /* All times in microseconds */
        /* A time_when_* refers to an instant in time, otherwise it refers to a length of time. */
 
        /* Master time statistics: */
        timestamp_t time_when_started; 
        timestamp_t time_send;         
        timestamp_t time_receive;      
        timestamp_t time_send_good;    
        timestamp_t time_receive_good; 
        timestamp_t time_status_msgs;  
        timestamp_t time_internal;     
        timestamp_t time_polling;      
        timestamp_t time_application;  
        timestamp_t time_scheduling;   
        /* Workers time statistics: */
        timestamp_t time_workers_execute;            
        timestamp_t time_workers_execute_good;       
        timestamp_t time_workers_execute_exhaustion; 
        /* BW statistics */
        int64_t bytes_sent;     
        int64_t bytes_received; 
        double  bandwidth;      
        /* resources statistics */
        int capacity_tasks;     
        int capacity_cores;     
        int capacity_memory;    
        int capacity_disk;      
        int capacity_gpus;      
        int capacity_instantaneous;      
        int capacity_weighted;  
        int64_t total_cores;      
        int64_t total_memory;     
        int64_t total_disk;           
        int64_t total_gpus;       
        int64_t committed_cores;  
        int64_t committed_memory; 
        int64_t committed_disk;   
        int64_t committed_gpus;   
        int64_t max_cores;        
        int64_t max_memory;       
        int64_t max_disk;         
        int64_t max_gpus;         
        int64_t min_cores;        
        int64_t min_memory;       
        int64_t min_disk;         
        int64_t min_gpus;         
        int64_t inuse_cache;     
};
 
 
struct vine_task *vine_task_create(const char *full_command);
 
void vine_task_delete(struct vine_task *t);
 
void vine_task_set_command( struct vine_task *t, const char *cmd );
 
void vine_task_needs_library( struct vine_task *t, const char *name );
 
void vine_task_provides_library( struct vine_task *t, const char *name );
 
 
void vine_task_set_function_slots( struct vine_task *t, int nslots );
 
 
int vine_task_add_input( struct vine_task *t, struct vine_file *f, const char *remote_name, vine_mount_flags_t flags );
 
int vine_task_add_output( struct vine_task *t, struct vine_file *f, const char *remote_name, vine_mount_flags_t flags );
 
void vine_task_set_retries( struct vine_task *t, int64_t max_retries );
 
void vine_task_set_max_forsaken( struct vine_task *t, int64_t max_forsaken );
 
void vine_task_set_memory( struct vine_task *t, int64_t memory );
 
void vine_task_set_disk( struct vine_task *t, int64_t disk );
 
void vine_task_set_cores( struct vine_task *t, int cores );
 
void vine_task_set_gpus( struct vine_task *t, int gpus );
 
void vine_task_set_time_end( struct vine_task *t, int64_t useconds );
 
void vine_task_set_time_start( struct vine_task *t, int64_t useconds );
 
void vine_task_set_time_max( struct vine_task *t, int64_t seconds );
 
void vine_task_set_time_min( struct vine_task *t, int64_t seconds );
 
void vine_task_set_tag(struct vine_task *t, const char *tag);
 
void vine_task_set_category(struct vine_task *t, const char *category);
 
void vine_task_add_feature(struct vine_task *t, const char *name);
 
void vine_task_set_priority(struct vine_task *t, double priority );
 
void vine_task_set_env_var( struct vine_task *t, const char *name, const char *value );
 
void vine_task_set_scheduler(struct vine_task *t, vine_schedule_t algorithm);
 
int vine_task_set_monitor_output(struct vine_task *t, const char *monitor_output);
 
const char * vine_task_get_command( struct vine_task *t );
 
const char * vine_task_get_tag( struct vine_task *t );
 
const char * vine_task_get_category( struct vine_task *t );
 
int vine_task_get_id( struct vine_task *t );
 
vine_result_t vine_task_get_result( struct vine_task *t );
 
const char *vine_result_string(vine_result_t result);
 
 
int vine_task_get_exit_code( struct vine_task *t );
 
const char * vine_task_get_stdout( struct vine_task *t );
 
const char * vine_task_get_addrport( struct vine_task *t );
 
const char * vine_task_get_hostname( struct vine_task *t );
 
int64_t vine_task_get_metric( struct vine_task *t, const char *name );
 
void vine_task_set_resources(struct vine_task *t, const struct rmsummary *rm );
 
const struct rmsummary *vine_task_get_resources( struct vine_task *t, const char *name );
 
int vine_task_set_snapshot_file(struct vine_task *t, struct vine_file *monitor_snapshot_file);
 
int vine_task_add_starch_package(struct vine_task *t, struct vine_file *f);
 
int vine_task_add_poncho_package(struct vine_task *t, struct vine_file *f);
 
int vine_task_add_execution_context(struct vine_task *t, struct vine_file *f);
 
/* Deprecated alias for vine_task_add_execution_context. */
int vine_task_add_environment(struct vine_task *t, struct vine_file *f);
 
 
 
const char * vine_file_contents( struct vine_file *f );
 
size_t vine_file_size( struct vine_file *f );
 
struct vine_file * vine_declare_file( struct vine_manager *m, const char *source, vine_cache_level_t cache, vine_file_flags_t flags );
 
struct vine_file * vine_declare_url( struct vine_manager *m, const char *url, vine_cache_level_t cache, vine_file_flags_t flags );
 
struct vine_file * vine_declare_xrootd( struct vine_manager *m, const char *source, struct vine_file *proxy, struct vine_file *env, vine_cache_level_t cache, vine_file_flags_t flags );
 
struct vine_file * vine_declare_chirp( struct vine_manager *m, const char *server, const char *source, struct vine_file *ticket, struct vine_file *env, vine_cache_level_t cache, vine_file_flags_t flags );
 
 
struct vine_file * vine_declare_temp( struct vine_manager *m );
 
struct vine_file * vine_declare_buffer( struct vine_manager *m, const char *buffer, size_t size, vine_cache_level_t cache, vine_file_flags_t flags );
 
struct vine_file *vine_declare_mini_task( struct vine_manager *m, struct vine_task *mini_task, const char *name, vine_cache_level_t cache, vine_file_flags_t flags);
 
struct vine_file * vine_declare_untar( struct vine_manager *m, struct vine_file *f, vine_cache_level_t cache, vine_file_flags_t flags);
 
struct vine_file * vine_declare_poncho( struct vine_manager *m, struct vine_file *f, vine_cache_level_t cache, vine_file_flags_t flags );
 
struct vine_file * vine_declare_starch( struct vine_manager *m, struct vine_file *f, vine_cache_level_t cache, vine_file_flags_t flags );
 
const char * vine_fetch_file( struct vine_manager *m, struct vine_file *f );
 
void vine_undeclare_file(struct vine_manager *m, struct vine_file *f );
 
 
 
struct vine_manager *vine_create(int port);
 
 
struct vine_manager *vine_ssl_create(int port, const char *key, const char *cert);
 
void vine_delete(struct vine_manager *m);
 
int vine_submit(struct vine_manager *m, struct vine_task *t);
 
void vine_manager_install_library( struct vine_manager *m, struct vine_task *t, const char *name );
 
void vine_manager_remove_library( struct vine_manager *m, const char *name );
 
struct vine_task *vine_wait(struct vine_manager *m, int timeout);
 
 
struct vine_task *vine_wait_for_tag(struct vine_manager *m, const char *tag, int timeout);
 
struct vine_task *vine_wait_for_task_id(struct vine_manager *m, int task_id, int timeout);
 
int vine_hungry(struct vine_manager *m);
 
int vine_empty(struct vine_manager *m);
 
int vine_port(struct vine_manager *m);
 
void vine_set_name(struct vine_manager *m, const char *name);
 
const char *vine_get_name(struct vine_manager *m);
 
int vine_enable_monitoring(struct vine_manager *m, int watchdog, int time_series);
 
int vine_enable_peer_transfers(struct vine_manager *m);
 
int vine_disable_peer_transfers(struct vine_manager *m);
 
int vine_set_task_id_min(struct vine_manager *m, int minid);
 
void vine_block_host(struct vine_manager *m, const char *hostname);
 
void vine_block_host_with_timeout(struct vine_manager *m, const char *hostname, time_t seconds);
 
 
void vine_unblock_host(struct vine_manager *m, const char *hostname);
 
void vine_unblock_all(struct vine_manager *m);
 
void vine_get_stats(struct vine_manager *m, struct vine_stats *s);
 
void vine_get_stats_category(struct vine_manager *m, const char *c, struct vine_stats *s);
 
 
char *vine_get_status(struct vine_manager *m, const char *request);
 
 
struct rmsummary **vine_summarize_workers(struct vine_manager *m);
 
void vine_set_bandwidth_limit(struct vine_manager *m, const char *bandwidth);
 
double vine_get_effective_bandwidth(struct vine_manager *m);
 
int vine_enable_disconnect_slow_workers(struct vine_manager *m, double multiplier);
 
 
int vine_enable_disconnect_slow_workers_category(struct vine_manager *m, const char *category, double multiplier);
 
 
int vine_set_draining_by_hostname(struct vine_manager *m, const char *hostname, int drain_flag);
 
int vine_set_category_mode(struct vine_manager *m, const char *category, vine_category_mode_t mode);
 
void vine_set_category_max_concurrent(struct vine_manager *m, const char *category, int max_concurrent);
 
int vine_enable_category_resource(struct vine_manager *m, const char *category, const char *resource, int autolabel);
 
void vine_set_scheduler(struct vine_manager *m, vine_schedule_t algorithm);
 
void vine_set_priority(struct vine_manager *m, int priority);
 
void vine_set_tasks_left_count(struct vine_manager *m, int ntasks);
 
void vine_set_catalog_servers(struct vine_manager *m, const char *hosts);
 
/* Send updates to the catalog server.
@param m A manager object
*/
void vine_update_catalog(struct vine_manager *m);
 
void vine_set_property( struct vine_manager *m, const char *name, const char *value );
        
int vine_cancel_by_task_id(struct vine_manager *m, int id);
 
int vine_cancel_by_task_tag(struct vine_manager *m, const char *tag);
 
int vine_cancel_all(struct vine_manager *m);
 
int vine_enable_debug_log( const char *logfile );
 
int vine_enable_perf_log(struct vine_manager *m, const char *logfile);
 
int vine_enable_transactions_log(struct vine_manager *m, const char *logfile);
 
int vine_enable_taskgraph_log(struct vine_manager *m, const char *logfile);
 
int vine_workers_shutdown(struct vine_manager *m, int n);
 
 
void vine_set_password( struct vine_manager *m, const char *password );
 
int vine_set_password_file( struct vine_manager *m, const char *file );
 
void vine_set_keepalive_interval(struct vine_manager *m, int interval);
 
void vine_set_keepalive_timeout(struct vine_manager *m, int timeout);
 
void vine_set_manager_preferred_connection(struct vine_manager *m, const char *preferred_connection);
 
int vine_tune(struct vine_manager *m, const char *name, double value);
 
void vine_set_resources_max(struct vine_manager *m,  const struct rmsummary *rm);
 
void vine_set_resources_min(struct vine_manager *m,  const struct rmsummary *rm);
 
void vine_set_category_resources_max(struct vine_manager *m,  const char *category, const struct rmsummary *rm);
 
void vine_set_category_resources_min(struct vine_manager *m,  const char *category, const struct rmsummary *rm);
 
void vine_set_category_first_allocation_guess(struct vine_manager *m,  const char *category, const struct rmsummary *rm);
 
void vine_initialize_categories(struct vine_manager *m, struct rmsummary *max, const char *summaries_file);
 
void vine_set_runtime_info_path(const char *path);
 
 
void vine_log_debug_app(struct vine_manager *q, const char *entry);
 
void vine_log_txn_app(struct vine_manager *q, const char *entry);
 
 
 
 
#endif