percpu.h

Go to the documentation of this file.

 
/*
 *  COPYRIGHT (c) 1989-2011.
 *  On-Line Applications Research Corporation (OAR).
 *
 *  Copyright (c) 2012, 2018 embedded brains GmbH
 *
 *  The license and distribution terms for this file may be
 *  found in the file LICENSE in this distribution or at
 *  http://www.rtems.org/license/LICENSE.
 */
 
#ifndef _RTEMS_PERCPU_H
#define _RTEMS_PERCPU_H
 
#include <rtems/score/cpuimpl.h>
 
#if defined( ASM )
  #include <rtems/asm.h>
#else
  #include <rtems/score/assert.h>
  #include <rtems/score/chain.h>
  #include <rtems/score/isrlock.h>
  #include <rtems/score/smp.h>
  #include <rtems/score/timestamp.h>
  #include <rtems/score/watchdog.h>
#endif
 
#ifdef __cplusplus
extern "C" {
#endif
 
#if defined(RTEMS_SMP)
  #if defined(RTEMS_PROFILING)
    #define PER_CPU_CONTROL_SIZE_APPROX \
      ( 512 + CPU_PER_CPU_CONTROL_SIZE + CPU_INTERRUPT_FRAME_SIZE )
  #elif defined(RTEMS_DEBUG) || CPU_SIZEOF_POINTER > 4
    #define PER_CPU_CONTROL_SIZE_APPROX \
      ( 256 + CPU_PER_CPU_CONTROL_SIZE + CPU_INTERRUPT_FRAME_SIZE )
  #else
    #define PER_CPU_CONTROL_SIZE_APPROX \
      ( 180 + CPU_PER_CPU_CONTROL_SIZE + CPU_INTERRUPT_FRAME_SIZE )
  #endif
 
  /*
   * This ensures that on SMP configurations the individual per-CPU controls
   * are on different cache lines to prevent false sharing.  This define can be
   * used in assembler code to easily get the per-CPU control for a particular
   * processor.
   */
  #if PER_CPU_CONTROL_SIZE_APPROX > 1024
    #define PER_CPU_CONTROL_SIZE_LOG2 11
  #elif PER_CPU_CONTROL_SIZE_APPROX > 512
    #define PER_CPU_CONTROL_SIZE_LOG2 10
  #elif PER_CPU_CONTROL_SIZE_APPROX > 256
    #define PER_CPU_CONTROL_SIZE_LOG2 9
  #elif PER_CPU_CONTROL_SIZE_APPROX > 128
    #define PER_CPU_CONTROL_SIZE_LOG2 8
  #else
    #define PER_CPU_CONTROL_SIZE_LOG2 7
  #endif
 
  #define PER_CPU_CONTROL_SIZE ( 1 << PER_CPU_CONTROL_SIZE_LOG2 )
#endif
 
#if !defined( ASM )
 
struct Record_Control;
 
struct _Thread_Control;
 
struct Scheduler_Context;
 
#if defined( RTEMS_SMP )
 
typedef enum {
  PER_CPU_STATE_INITIAL,
 
  PER_CPU_STATE_READY_TO_START_MULTITASKING,
 
  PER_CPU_STATE_REQUEST_START_MULTITASKING,
 
  PER_CPU_STATE_UP,
 
  PER_CPU_STATE_SHUTDOWN
} Per_CPU_State;
 
typedef void ( *Per_CPU_Job_handler )( void *arg );
 
typedef struct {
  Per_CPU_Job_handler handler;
 
  void *arg;
} Per_CPU_Job_context;
 
/*
 * Value for the Per_CPU_Job::done member to indicate that a job is done
 * (handler was called on the target processor).  Must not be a valid pointer
 * value since it overlaps with the Per_CPU_Job::next member.
 */
#define PER_CPU_JOB_DONE 1
 
typedef struct Per_CPU_Job {
  union {
    struct Per_CPU_Job *next;
 
    Atomic_Ulong done;
  };
 
  const Per_CPU_Job_context *context;
} Per_CPU_Job;
 
#endif /* defined( RTEMS_SMP ) */
 
typedef struct {
#if defined( RTEMS_PROFILING )
  CPU_Counter_ticks thread_dispatch_disabled_instant;
 
  CPU_Counter_ticks max_thread_dispatch_disabled_time;
 
  CPU_Counter_ticks max_interrupt_time;
 
  CPU_Counter_ticks max_interrupt_delay;
 
  uint64_t thread_dispatch_disabled_count;
 
  uint64_t total_thread_dispatch_disabled_time;
 
  uint64_t interrupt_count;
 
  uint64_t total_interrupt_time;
#endif /* defined( RTEMS_PROFILING ) */
} Per_CPU_Stats;
 
typedef enum {
  PER_CPU_WATCHDOG_TICKS,
 
  PER_CPU_WATCHDOG_REALTIME,
 
  PER_CPU_WATCHDOG_MONOTONIC,
 
  PER_CPU_WATCHDOG_COUNT
} Per_CPU_Watchdog_index;
 
typedef struct Per_CPU_Control {
  #if CPU_PER_CPU_CONTROL_SIZE > 0
    CPU_Per_CPU_control cpu_per_cpu;
  #endif
 
  void *interrupt_stack_low;
 
  void *interrupt_stack_high;
 
  uint32_t isr_nest_level;
 
  uint32_t isr_dispatch_disable;
 
  volatile uint32_t thread_dispatch_disable_level;
 
  volatile bool dispatch_necessary;
 
  /*
   * Ensure that the executing member is at least 4-byte aligned, see
   * PER_CPU_OFFSET_EXECUTING.  This is necessary on CPU ports with relaxed
   * alignment restrictions, e.g. type alignment is less than the type size.
   */
  bool reserved_for_executing_alignment[ 3 ];
 
  struct _Thread_Control *executing;
 
  struct _Thread_Control *heir;
 
#if defined(RTEMS_SMP)
  CPU_Interrupt_frame Interrupt_frame;
#endif
 
  Timestamp_Control cpu_usage_timestamp;
 
  struct {
    ISR_LOCK_MEMBER( Lock )
 
    
    uint64_t ticks;
 
    Watchdog_Header Header[ PER_CPU_WATCHDOG_COUNT ];
  } Watchdog;
 
  #if defined( RTEMS_SMP )
    ISR_lock_Control Lock;
 
    ISR_lock_Context Lock_context;
 
    Chain_Control Threads_in_need_for_help;
 
    Atomic_Ulong message;
 
    struct {
      const struct _Scheduler_Control *control;
 
      const struct Scheduler_Context *context;
 
      struct _Thread_Control *idle_if_online_and_unused;
    } Scheduler;
 
    struct _Thread_Control *ancestor;
 
    char *data;
 
    Per_CPU_State state;
 
    struct {
      ISR_lock_Control Lock;
 
      struct Per_CPU_Job *head;
 
      struct Per_CPU_Job **tail;
    } Jobs;
 
    bool online;
 
    bool boot;
  #endif
 
  struct Record_Control *record;
 
  Per_CPU_Stats Stats;
} Per_CPU_Control;
 
#if defined( RTEMS_SMP )
typedef struct {
  Per_CPU_Control per_cpu;
  char unused_space_for_cache_line_alignment
    [ PER_CPU_CONTROL_SIZE - sizeof( Per_CPU_Control ) ];
} Per_CPU_Control_envelope;
#else
typedef struct {
  Per_CPU_Control per_cpu;
} Per_CPU_Control_envelope;
#endif
 
extern Per_CPU_Control_envelope _Per_CPU_Information[] CPU_STRUCTURE_ALIGNMENT;
 
#define _Per_CPU_Acquire( cpu, lock_context ) \
  _ISR_lock_Acquire( &( cpu )->Lock, lock_context )
 
#define _Per_CPU_Release( cpu, lock_context ) \
  _ISR_lock_Release( &( cpu )->Lock, lock_context )
 
/*
 * If we get the current processor index in a context which allows thread
 * dispatching, then we may already run on another processor right after the
 * read instruction.  There are very few cases in which this makes sense (here
 * we can use _Per_CPU_Get_snapshot()).  All other places must use
 * _Per_CPU_Get() so that we can add checks for RTEMS_DEBUG.
 */
#if defined( _CPU_Get_current_per_CPU_control )
  #define _Per_CPU_Get_snapshot() _CPU_Get_current_per_CPU_control()
#else
  #define _Per_CPU_Get_snapshot() \
    ( &_Per_CPU_Information[ _SMP_Get_current_processor() ].per_cpu )
#endif
 
#if defined( RTEMS_SMP )
static inline Per_CPU_Control *_Per_CPU_Get( void )
{
  Per_CPU_Control *cpu_self = _Per_CPU_Get_snapshot();
 
  _Assert(
    cpu_self->thread_dispatch_disable_level != 0 || _ISR_Get_level() != 0
  );
 
  return cpu_self;
}
#else
#define _Per_CPU_Get() _Per_CPU_Get_snapshot()
#endif
 
static inline Per_CPU_Control *_Per_CPU_Get_by_index( uint32_t index )
{
  return &_Per_CPU_Information[ index ].per_cpu;
}
 
static inline uint32_t _Per_CPU_Get_index( const Per_CPU_Control *cpu )
{
#if defined(RTEMS_SMP)
  const Per_CPU_Control_envelope *per_cpu_envelope =
    ( const Per_CPU_Control_envelope * ) cpu;
 
  return ( uint32_t ) ( per_cpu_envelope - &_Per_CPU_Information[ 0 ] );
#else
  (void) cpu;
  return 0;
#endif
}
 
static inline struct _Thread_Control *_Per_CPU_Get_executing(
  const Per_CPU_Control *cpu
)
{
  return cpu->executing;
}
 
static inline bool _Per_CPU_Is_processor_online(
  const Per_CPU_Control *cpu
)
{
#if defined( RTEMS_SMP )
  return cpu->online;
#else
  (void) cpu;
 
  return true;
#endif
}
 
static inline bool _Per_CPU_Is_boot_processor(
  const Per_CPU_Control *cpu
)
{
#if defined( RTEMS_SMP )
  return cpu->boot;
#else
  (void) cpu;
 
  return true;
#endif
}
 
RTEMS_INLINE_ROUTINE void _Per_CPU_Acquire_all(
  ISR_lock_Context *lock_context
)
{
#if defined(RTEMS_SMP)
  uint32_t         cpu_max;
  uint32_t         cpu_index;
  Per_CPU_Control *previous_cpu;
 
  cpu_max = _SMP_Get_processor_maximum();
  previous_cpu = _Per_CPU_Get_by_index( 0 );
 
  _ISR_lock_ISR_disable( lock_context );
  _Per_CPU_Acquire( previous_cpu, lock_context );
 
  for ( cpu_index = 1 ; cpu_index < cpu_max ; ++cpu_index ) {
     Per_CPU_Control *cpu;
 
     cpu = _Per_CPU_Get_by_index( cpu_index );
     _Per_CPU_Acquire( cpu, &previous_cpu->Lock_context );
     previous_cpu = cpu;
  }
#else
  _ISR_lock_ISR_disable( lock_context );
#endif
}
 
RTEMS_INLINE_ROUTINE void _Per_CPU_Release_all(
  ISR_lock_Context *lock_context
)
{
#if defined(RTEMS_SMP)
  uint32_t         cpu_max;
  uint32_t         cpu_index;
  Per_CPU_Control *cpu;
 
  cpu_max = _SMP_Get_processor_maximum();
  cpu = _Per_CPU_Get_by_index( cpu_max - 1 );
 
  for ( cpu_index = cpu_max - 1 ; cpu_index > 0 ; --cpu_index ) {
     Per_CPU_Control *previous_cpu;
 
     previous_cpu = _Per_CPU_Get_by_index( cpu_index - 1 );
     _Per_CPU_Release( cpu, &previous_cpu->Lock_context );
     cpu = previous_cpu;
  }
 
  _Per_CPU_Release( cpu, lock_context );
  _ISR_lock_ISR_enable( lock_context );
#else
  _ISR_lock_ISR_enable( lock_context );
#endif
}
 
#if defined( RTEMS_SMP )
 
void _Per_CPU_Initialize(void);
 
void _Per_CPU_State_change(
  Per_CPU_Control *cpu,
  Per_CPU_State new_state
);
 
bool _Per_CPU_State_wait_for_non_initial_state(
  uint32_t cpu_index,
  uint32_t timeout_in_ns
);
 
void _Per_CPU_Perform_jobs( Per_CPU_Control *cpu );
 
void _Per_CPU_Add_job( Per_CPU_Control *cpu, Per_CPU_Job *job );
 
void _Per_CPU_Wait_for_job(
  const Per_CPU_Control *cpu,
  const Per_CPU_Job     *job
);
 
#endif /* defined( RTEMS_SMP ) */
 
/*
 * On a non SMP system, the _SMP_Get_current_processor() is defined to 0.
 * Thus when built for non-SMP, there should be no performance penalty.
 */
#define _Thread_Dispatch_disable_level \
  _Per_CPU_Get()->thread_dispatch_disable_level
#define _Thread_Heir \
  _Per_CPU_Get()->heir
 
#if defined(_CPU_Get_thread_executing)
#define _Thread_Executing \
  _CPU_Get_thread_executing()
#else
#define _Thread_Executing \
  _Per_CPU_Get_executing( _Per_CPU_Get() )
#endif
 
#define _ISR_Nest_level \
  _Per_CPU_Get()->isr_nest_level
#define _CPU_Interrupt_stack_low \
  _Per_CPU_Get()->interrupt_stack_low
#define _CPU_Interrupt_stack_high \
  _Per_CPU_Get()->interrupt_stack_high
#define _Thread_Dispatch_necessary \
  _Per_CPU_Get()->dispatch_necessary
 
RTEMS_INLINE_ROUTINE struct _Thread_Control *_Thread_Get_executing( void )
{
  struct _Thread_Control *executing;
 
  #if defined(RTEMS_SMP) && !defined(_CPU_Get_thread_executing)
    ISR_Level level;
 
    _ISR_Local_disable( level );
  #endif
 
  executing = _Thread_Executing;
 
  #if defined(RTEMS_SMP) && !defined(_CPU_Get_thread_executing)
    _ISR_Local_enable( level );
  #endif
 
  return executing;
}
 
#endif /* !defined( ASM ) */
 
#if defined( ASM ) || defined( _RTEMS_PERCPU_DEFINE_OFFSETS )
 
#define PER_CPU_INTERRUPT_STACK_LOW \
  CPU_PER_CPU_CONTROL_SIZE
#define PER_CPU_INTERRUPT_STACK_HIGH \
  PER_CPU_INTERRUPT_STACK_LOW + CPU_SIZEOF_POINTER
 
#define INTERRUPT_STACK_LOW \
  (SYM(_Per_CPU_Information) + PER_CPU_INTERRUPT_STACK_LOW)
#define INTERRUPT_STACK_HIGH \
  (SYM(_Per_CPU_Information) + PER_CPU_INTERRUPT_STACK_HIGH)
 
/*
 *  These are the offsets of the required elements in the per CPU table.
 */
#define PER_CPU_ISR_NEST_LEVEL \
  PER_CPU_INTERRUPT_STACK_HIGH + CPU_SIZEOF_POINTER
#define PER_CPU_ISR_DISPATCH_DISABLE \
  PER_CPU_ISR_NEST_LEVEL + 4
#define PER_CPU_THREAD_DISPATCH_DISABLE_LEVEL \
  PER_CPU_ISR_DISPATCH_DISABLE + 4
#define PER_CPU_DISPATCH_NEEDED \
  PER_CPU_THREAD_DISPATCH_DISABLE_LEVEL + 4
#define PER_CPU_OFFSET_EXECUTING \
  PER_CPU_DISPATCH_NEEDED + 4
#define PER_CPU_OFFSET_HEIR \
  PER_CPU_OFFSET_EXECUTING + CPU_SIZEOF_POINTER
#if defined(RTEMS_SMP)
#define PER_CPU_INTERRUPT_FRAME_AREA \
  PER_CPU_OFFSET_HEIR + CPU_SIZEOF_POINTER
#endif
 
#define THREAD_DISPATCH_DISABLE_LEVEL \
  (SYM(_Per_CPU_Information) + PER_CPU_THREAD_DISPATCH_DISABLE_LEVEL)
#define ISR_NEST_LEVEL \
  (SYM(_Per_CPU_Information) + PER_CPU_ISR_NEST_LEVEL)
#define DISPATCH_NEEDED \
  (SYM(_Per_CPU_Information) + PER_CPU_DISPATCH_NEEDED)
 
#endif /* defined( ASM ) || defined( _RTEMS_PERCPU_DEFINE_OFFSETS ) */
 
#ifdef __cplusplus
}
#endif
 
#endif
/* end of include file */