#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include "hibutil.h"
#include "singutil.h"

static void init_envs(void);
static int  sizeof_i_particle_vector_struct(void);
static int sizeof_final_out(int final_id);

static void em_write2(int devid, int nidp, int adrem);
static void run_counter(int devid, int nisp,int adrisp,int nrepisp,
                        int ncntidp,int nidp,int adridp, int memadr,int nrepidp);
static void em_write(int devid, int nidp, int adrem);
static void send_ispdata(int devid, int n, int ibegin, UINT64 ispmdata[],UINT64 ispldata[]);
static void gdrtb_sendMC(int devid, int size, UINT64 *buf);
static void send_vector(int devid, int i0, int i1, UINT64 idp0data[], UINT64 idp1data[]);

// -----------------------------------------------
//(need multi-device treatment: BEGIN)

static int Ndevice = 0;
static int Devid[NHIBMAX]; // Devid[0..Ndevice-1] stores device IDs in use.
static Devpara Dpara = J_DEVPARA;

int SING_ndevice(void)
{
    init_envs();
    return Ndevice;
}

void SING_set_devpara(Devpara dpara)
{
    init_envs();
    Dpara = dpara;
}

Devpara SING_devpara(void)
{
    return Dpara;
}

void SING_open(void)
{
    int ic;

    init_envs();
    for (ic = 0; ic < Ndevice; ic++) {
        SING_openMC(Devid[ic]);
    }
}

void SING_close(void)
{
    int ic;

    for (ic = 0; ic < Ndevice; ic++) {
        SING_closeMC(Devid[ic]);
    }
}

void SING_LM_write_regconv(int length, int *conversions)
{
    int ic;

    for (ic = 0; ic < Ndevice; ic++) {
        SING_LM_write_regconvMC(Devid[ic], length, conversions);
    }
}

/*
 * data size to be sent to LM : (UINT64 x nword)          if peeach == 0,
 *                              (UINT64 x nword * peeach) otherwise.
 * note that nword must be set by SING_LM_write_regconvMC() in advance.
 *
 * hostp    : data to be sent.
 * lmoffset : destination address in LM space (in byte?).
 * peeach   : number of i_particle_vector_structs. 0 for constant.
 */
void SING_LM_write(void *hostp, int lmoffset, int peeach)
{
    int ic;

    if (Dpara == J_DEVPARA || peeach == 0) {
        for (ic = 0; ic < Ndevice; ic++) {
            SING_LM_writeMC(Devid[ic], hostp, lmoffset, peeach);
        }
    }
    else { // i-parallelism.
        char *srcaddr = hostp;
        int nip = (peeach - 1) / Ndevice + 1; // number of i_particle_vector_structs per device.
        int addrinc = sizeof_i_particle_vector_struct() * nip;
        for (ic = 0; ic < Ndevice; ic++) {
            SING_LM_writeMC(Devid[ic], srcaddr, lmoffset, nip);
            srcaddr += addrinc;
        }
    }
}

int SING_EM_alloc(int length, int *conversions, int size, int *err)
{
    int ic, retval;

    for (ic = 0; ic < Ndevice; ic++) {
        retval = SING_EM_allocMC(Devid[ic], length, conversions, size, err);
    }
    return retval;
}

/*
 * data size to be sent to EM : (UINT64 x nword * ndata)
 *                             
 * note that nword must be set by SING_LM_write_regconvMC() in advance.
 *
 * areaid   : ?
 * start    : destination address in EM space (in JP-size unit).
 * ndata    : number of j_particle_structs. **SHOULD ALWAYS BE 1**
 * hostp    : data to be sent.
 */
int SING_EM_send(int areaid, int start, int ndata, void *hostp)
{
    int ic, dstaddr, retval = 0;

    assert(ndata == 1);

    if (Dpara == J_DEVPARA) {
        ic = start % Ndevice; // destination device.
        dstaddr = start / Ndevice;
        retval = SING_EM_sendMC(Devid[ic], areaid, dstaddr, ndata, hostp);
    }
    else { // i-parallelism.
        for (ic = 0; ic < Ndevice; ic++) {
            retval |= SING_EM_sendMC(Devid[ic], areaid, start, ndata, hostp);
        }
    }

    return retval; // 0:normal  1:error
}

/*
 * data size to be sent to EM : (UINT64 x nword * ndata)
 * note that nword must be set by SING_LM_write_regconvMC() in advance.
 *
 * areaid   : ?
 * start    : destination address in EM space (in JP-size unit).
 * ndata    : number of j_particle_structs. **SHOULD ALWAYS BE 16**
 * hostp    : data to be sent.
 */
void SING_EM_write(int areaid, int start, int ndata, void *hostp)
{
    int ic, dstaddr;
    int nbb = SING_number_of_bb();

    assert(ndata == nbb);

    if (Dpara == J_DEVPARA) {
        ic = (start / nbb) % Ndevice; // destination device.
        dstaddr = start / nbb / Ndevice * nbb;
        SING_EM_writeMC(Devid[ic], areaid, dstaddr, ndata, hostp);
    }
    else { // i-parallelism.
        for (ic = 0; ic < Ndevice; ic++) {
            SING_EM_writeMC(Devid[ic], areaid, start, ndata, hostp);
        }
    }
}

void SING_iflush(void)
{
    int ic;

    for (ic = 0; ic < Ndevice; ic++) {
        SING_iflushMC(Devid[ic]);
    }
}
     
/*
 * data size received : (UINT64 x nword * ndata)
 *
 * loop_count  : 
 * loop_out    : not used.
 * final_out   : data received.
 */
void SING_execute(int init_id, int loop_id, int final_id, int loop_count, 
                 void *loop_out, void *final_out)
{
    enum {
        FOBUFSIZE = 32*4*1024, // 1k double vars per result_struct should be enough.
    };
    int ic, i, off, lc;
    double *final_out64 = (double *)final_out;
    static double lobuf[NHIBMAX][256]; // not used. just a dummy.
    static double fobuf[NHIBMAX][FOBUFSIZE];

#pragma omp parallel for private (lc)
    for (ic = 0; ic < Ndevice; ic++) {
        lc = loop_count / Ndevice;
        if (loop_count % Ndevice > ic) {
            lc++;
        }
        SING_executeMC(Devid[ic], init_id, loop_id, final_id, lc, lobuf[ic], fobuf[ic]);
    }

    int fobufsize = sizeof_final_out(final_id) / sizeof(double); // size of fobuf in double.

    if (Dpara == J_DEVPARA) { // fobuf[]s are summed up.
        for (i = 0; i < fobufsize; i++) {
            final_out64[i] = 0.0;
        }
        for (ic = 0; ic < Ndevice; ic++) {
            for (i = 0; i < fobufsize; i++) {
                final_out64[i] += fobuf[ic][i];
            }
        }
    }
    else { // i-parallelism. fobuf[]s are concatenated.
        off = 0;
        for (ic = 0; ic < Ndevice; ic++) {
            for (i = 0; i < fobufsize; i++) {
                final_out64[off] = fobuf[ic][i];
                off++;
            }
        }
    }
}

int SING_iregister(int ninsts, char **insts)
{
    int ic, retval;

    for (ic = 0; ic < Ndevice; ic++) {
        retval = SING_iregisterMC(Devid[ic], ninsts, insts);
    }
    return retval;
}

static void init_envs(void)
{
    int ic;
    int dummy[NHIBMAX];
    char *p;
    char *cardno;
    static int firstcall = 1;

    if (Ndevice != 0) {
        return;
    }

    /*
     * cards are not allocated yet.
     * try to allocate cards specified by environment variable "GDEVICE".
     * try to allocate all cards, if GDEVICE is not set.
     */
    p = getenv("GDEVICE");
    if (p) { // parse GDEVICE
        static int device_used[NHIBMAX]; // 1:in use.  0:not used.
        for (ic = 0; ic < hib_ndevice(); ic++) {
            device_used[ic] = 0;
        }
        cardno = strtok(p, " ");
        while (cardno) {
            ic = atoi(cardno);
            if (ic < 0 || ic >= hib_ndevice()) {
                fprintf(stderr, "GDEVICE have device_id out of range: %d\n", ic);
                exit(2);
            }
            if (!device_used[ic]) { // care for malformed GDEVICE value such as "0 1 2 2 3".
                device_used[ic] = 1;
                Devid[Ndevice] = ic;
                Ndevice++;
            }
            cardno = strtok(NULL, " ");
        }
    }
    else { // GDEVICE is not set
        Ndevice = hib_ndevice();
        for (ic = 0; ic < hib_ndevice(); ic++) {
            Devid[ic] = ic;
        }
    }

    p = getenv("GDEVPARA");
    if (p) {
        if (0 == strcmp("I", p) || 0 == strcmp("i", p)) {
            SING_set_devpara(I_DEVPARA);
            fprintf(stderr, "Device parallelism set to i.\n");
        }
        else if (0 == strcmp("J", p) || 0 == strcmp("j", p)) {
            SING_set_devpara(J_DEVPARA);
            fprintf(stderr, "Device parallelism set to j.\n");
        }
        else {
            fprintf(stderr, "Invalid GDEVPARA. Device parallelism not changed.\n");
        }
    }
}

void SING_set_result_size(int unkwn1, int unkwn2)
{
}

//(need multi-device treatment: END)
//------------------------------------------------

int SING_number_of_pe(void)
{
  return 32;
}

int SING_number_of_bb(void)
{
  return 16;
}

// -----------------------------------------------

int SING_EM_BM_transfer(int areaid, int emstart, int bmstart, int ndata, int mode, int bbid)
{
  return 1;
}

int SING_BM_alloc(int areaid, int size, int *err)
{
  return 1;
}

// -----------------------------------------------

static Hib *h[NHIBMAX];
static UINT64 *rbuf[NHIBMAX];
static UINT64 *wbuf[NHIBMAX];
static UINT64 piowbuf[NHIBMAX][1024];
static int Model[NHIBMAX];

static void reset_odp(int clusterid)
{
  rbuf[clusterid][0] = 0x0000000b00000007ll;
  rbuf[clusterid][1] = 0x0000000000000000ll;
  rbuf[clusterid][2] = 0x0000000000000000ll;
  rbuf[clusterid][3] = 0x0000000000000007ll;
  rbuf[clusterid][4] = 0x0000000000000007ll;
  rbuf[clusterid][5] = 0x0000000000000007ll;
  rbuf[clusterid][6] = 0x0000000000000007ll;
  rbuf[clusterid][7] = 0x0000000000000000ll;

  hib_sendMC(clusterid, 8, rbuf[clusterid]);
}

static void reset_hib(int clusterid)
{
  if (h[clusterid]->type == HIB_GRAPEDRG) {
    hib_mem_writeMC(clusterid, h[clusterid]->r->dma0cmd, (1 << h[clusterid]->r->dma0cmd_abort_bit));
    hib_mem_writeMC(clusterid, h[clusterid]->r->dma1cmd, (1 << h[clusterid]->r->dma1cmd_abort_bit));
  }
  else {
    hib_mem_writeMC(clusterid, h[clusterid]->r->command, (1<<h[clusterid]->r->command_dma_reset_bit));
  }
}

void SING_openMC(int devid)
{
  UINT32 binfo,productid,modelid;

  h[devid] = hib_openMC(devid);

  hib_set_sendfuncMC(devid, SENDFUNC_PIOW);
  rbuf[devid] = piowbuf[devid];
  wbuf[devid] = h[devid]->dmaw_buf;

  reset_hib(devid);

  binfo = hib_mem_readMC(devid, h[devid]->r->boardinfo);
  productid = (binfo >> 28) & 0xf;
  modelid = (binfo >> 24) & 0xf;
  if(modelid==3) Model[devid] = 460;
  if(modelid==4) Model[devid] = 460;  

  reset_odp(devid);
  reset_hib(devid);
  reset_hib(devid);  
  reset_hib(devid);

}

void SING_closeMC(int devid)
{
  hib_closeMC(devid);
}

static int nconv_table[NHIBMAX];
static int nword[NHIBMAX];
static int conv_table[NHIBMAX][1000];
static UINT64 idp00[NHIBMAX][10000];
static UINT64 idp01[NHIBMAX][10000];
static UINT64 idp10[NHIBMAX][10000];
static UINT64 idp11[NHIBMAX][10000];
static UINT64 ispm0[NHIBMAX][10000];
static UINT64 ispl0[NHIBMAX][10000];
static UINT64 idp0[NHIBMAX][10000];
static UINT64 idp1[NHIBMAX][10000];

void SING_LM_write_regconvMC(int devid, int length, int *conversions)
{
  int i,wordhalf;

  nword[devid] = 0; wordhalf=0;
  nconv_table[devid] = length;
  for(i=0;i<length;i++) conv_table[devid][i] = conversions[i];

  for(i=0;i<length;i++){
    if((conv_table[devid][i] == 0)||(conv_table[devid][i] == 4)){
      if(wordhalf==1){
	wordhalf = 0;   // vsm arrnge alignment
	nword[devid]++;
      }
      nword[devid]++;
    }
    if((conv_table[devid][i] == 1)||(conv_table[devid][i] == 6)||(conv_table[devid][i] == 9)){
      if(wordhalf==1){
	nword[devid]++;
	wordhalf = 0;
      }else{
	wordhalf = 1;
      }
    }
    if((conv_table[devid][i] == 2)||(conv_table[devid][i] == 3)||(conv_table[devid][i] == 5)||(conv_table[devid][i] == 7)||(conv_table[devid][i] == 8)){
      fprintf(stderr,"SING_LM_write_regconv: current version does not support conversion=%d\n",conv_table[devid][i]);
    }
  }
  if(wordhalf == 1) nword[devid]++;
  //  printf("nword[devid] %d\n",nword[devid]);
}

void SING_LM_writeMC(int devid, void *hostp, int lmoffset, int peeach)
{
  int i,ii,i0,i1,nloop,iii,ipe;
  UINT64 tmp,tmp2;
  
  for(i=0;i<nconv_table[devid];i++){
    tmp = *(((UINT64 *)hostp)+i);
    //    printf("conv_table %d %d %016llx\n",i,conv_table[devid][i],tmp);
  }
  
  if(peeach == 0){  // constant 

    idp0[devid][0] = 0x0000000000000000 | ((UINT64)nword[devid])<<56 ;  idp1[devid][0] = 0x1;

    iii=0;
    for(i=0;i<nword[devid];i++){

      if(conv_table[devid][iii]==1){
        tmp = *(((UINT64 *)hostp)+iii);
	if(((iii+1)==nconv_table[devid])||(conv_table[devid][iii+1]==4)){
          idp0[devid][i+1] = (0xfffffffff0000000 & tmp);
	  idp1[devid][i+1] = 0x1;
   	  iii++;
	}else if(conv_table[devid][iii+1]==1){
          tmp2 = *(((UINT64 *)hostp)+iii+1);	
          idp0[devid][i+1] = (0xfffffffff0000000 & tmp) | (0xfffffff & (tmp2 >> 36)) ;
  	  idp1[devid][i+1] = (0xff&(tmp2>>28))<<9 | 0x1;
   	  iii+=2;
	}else if(conv_table[devid][iii+1]==6){
          tmp2 = *(((UINT64 *)hostp)+iii+1);	
          idp0[devid][i+1] = (0xfffffffff0000000 & tmp) | ((0xfffffffff & tmp2) >> 8) ;
          idp1[devid][i+1] = ((0xff&tmp2)<<9) | 0x1;
   	  iii+=2;
	}

      }else if(conv_table[devid][iii]==4){
        tmp = *(((UINT64 *)hostp)+iii);
	if((0x1&(tmp>>63))==1){
          idp0[devid][i+1] = 0xff00000000000000 | tmp>>8; 
	}else{
          idp0[devid][i+1] = tmp>>8; 
	}
        idp1[devid][i+1] = (0xff&tmp)<<9 | 0x1;
        iii++;

      }else if(conv_table[devid][iii]==0){
        idp0[devid][i+1] = *(((UINT64 *)hostp)+iii);
        idp1[devid][i+1] = 0x1;
        iii++;

      }else if(conv_table[devid][iii]==6){
        tmp = *(((UINT64 *)hostp)+iii);
	if(((iii+1)==nconv_table[devid])||(conv_table[devid][iii+1]==4)){
          idp0[devid][i+1] = (0xfffffffff & tmp) << 28 ;
          idp1[devid][i+1] = 0x1;
   	  iii++;
	}else if(conv_table[devid][iii+1]==6){
          tmp2 = *(((UINT64 *)hostp)+iii+1);	
          idp0[devid][i+1] = (0xfffffffff & tmp) << 28 | (0xfffffffff & tmp2 ) >> 8 ;
          idp1[devid][i+1] = (0xff&tmp2)<<9 | 0x1;
  	  iii+=2;
	}else if(conv_table[devid][iii+1]==1){
          tmp2 = *(((UINT64 *)hostp)+iii+1);	
          idp0[devid][i+1] = (0xfffffffff & tmp) << 28 | (0xfffffff & (tmp2 >> 36)) ;
          idp1[devid][i+1] = (0xff&(tmp2>>28))<<9 | 0x1;
   	  iii+=2;
	}

      }

    }

    for(i=0;i<(nword[devid]+1);i++){
      //      printf("ii %d idp0,1 %016llx %05llx\n",i,idp0[devid][i],idp1[devid][i]);
    }

    i0 =0; i1=nword[devid];
    send_vector(devid,i0,i1,idp0[devid],idp1[devid]);

    ii = 0;
    for(i=0;i<nword[devid];i+=4){
      if((i+4)>nword[devid]){
	nloop = nword[devid] - i;
      }else{
	nloop = 4;
      }
      ispm0[devid][ii] = 0x00000000006000a0 | ((UINT64)nloop)<<61 | ((UINT64)(lmoffset+2*i))<<11;
      ispl0[devid][ii] = 0x0000000000100000 | ((UINT64)(2*i))<<26 ;
//      printf("i %d ispm,l %016llx %016llx\n",i,ispm0[devid][ii],ispl0[devid][ii]);
      ii++;
    }
    send_ispdata(devid,ii,0,ispm0[devid],ispl0[devid]);
    run_counter(devid,2*ii,0,1,0,0,0,0,0);                  

  }else{  // i-particle

    idp0[devid][0] = 0x0000000000000000 | ((UINT64)nword[devid])<<56 ;  idp1[devid][0] = 0x1;

    ii=0;
    for(ipe=0;ipe<peeach;ipe++){

      iii=0; 
      for(i=0;i<nword[devid];i++){

        if(conv_table[devid][iii]==0){
          tmp = *(((UINT64 *)hostp)+ii);
          idp0[devid][i+1] = tmp; idp1[devid][i+1] = 0x1;
          iii++; ii++;

	}else if(conv_table[devid][iii]==1){
          tmp = *(((UINT64 *)hostp)+ii);
  	  if((iii+1)==nconv_table[devid]){
	    tmp2 = 0;
	  }else{
            tmp2 = *(((UINT64 *)hostp)+ii+1);	
	  }
          idp0[devid][i+1] = (0xfffffffff0000000 & tmp) | (0xfffffff & (tmp2 >> 36)) ;
	  idp1[devid][i+1] = (0xff&(tmp2>>28))<<9 | 0x1;
	  iii+=2; ii+=2; 

        }else if(conv_table[devid][iii]==9){
          tmp = *(((UINT64 *)hostp)+ii);
          idp0[devid][i+1] = (0xffffffff & tmp ) << 28 | (0xffffffff00000000 & tmp) >> 40 ;  // endian
          idp1[devid][i+1] = (0xff&(tmp>>32))<<9 | 0x1;
 	  iii+=2; ii++;
	}

      }

      for(i=0;i<(nword[devid]+1);i++){
	//        printf("ipe %d ii %d idp0,1 %016llx %05llx\n",ipe,i,idp0[devid][i],idp1[devid][i]);
      }

      i0 =0; i1=nword[devid];
      send_vector(devid,i0,i1,idp0[devid],idp1[devid]);

      if(ipe==0){
        ispm0[devid][0] = 0x8024000000000000; ispl0[devid][0] = 0x0001470000000000;
        ispm0[devid][1] = 0x8024000000000000; ispl0[devid][1] = 0x0001470000000000;
        ispm0[devid][2] = 0x8080000000000000; ispl0[devid][2] = 0x0002470000000000;
        ispm0[devid][3] = 0x8024000000000000; ispl0[devid][3] = 0x0001420000000000;
        iii = 4;
      }else{
        ispm0[devid][0] = 0x8080000000000000; ispl0[devid][0] = 0x0002470000000000;
        ispm0[devid][1] = 0x8024000000000000; ispl0[devid][1] = 0x0001420000000000;
        iii = 2;
      }

      for(i=0;i<nword[devid];i+=4){
        if((i+4)>nword[devid]){
  	  nloop = nword[devid] - i;
        }else{
	  nloop = 4;
        }
        ispm0[devid][iii] = 0x04000000006000a0 | ((UINT64)nloop)<<61 | ((UINT64)(lmoffset+2*i))<<11;
        ispl0[devid][iii] = 0x0000000000100000 | ((UINT64)(2*i))<<26 ;
//        printf("i %d ispm,l %016llx %016llx\n",iii,ispm0[devid][iii],ispl0[devid][iii]);
        iii++;
      }

      send_ispdata(devid,iii,0,ispm0[devid],ispl0[devid]);
      run_counter(devid,2*iii,0,1,0,0,0,0,0);                  
    }

  }

}

//

static int conv_tableEM[NHIBMAX][1000];
static int nconv_tableEM[NHIBMAX];
static int nwordEM[NHIBMAX];
static int areaid[NHIBMAX]={0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};

int SING_EM_allocMC(int devid, int length, int *conversions, int size, int *err)
{
  int i,wordhalf;

  nwordEM[devid] = 0; wordhalf=0;
  nconv_tableEM[devid] = length;
  for(i=0;i<length;i++) conv_tableEM[devid][i] = conversions[i];

  for(i=0;i<length;i++){

    if(conv_tableEM[devid][i] == 0){
      if(wordhalf==1){
	fprintf(stderr,"SING_EM_alloc: alignment error");
	exit(1);
      }else{
	nwordEM[devid]++;
      }
    }

    if((conv_tableEM[devid][i] == 1)||(conv_tableEM[devid][i] == 9)){
      if(wordhalf==1){
	nwordEM[devid]++;
	wordhalf = 0;
      }else{
	wordhalf = 1;
      }
    }

    if((conv_tableEM[devid][i] == 2)||(conv_tableEM[devid][i] == 3)||(conv_tableEM[devid][i] == 4)
     ||(conv_tableEM[devid][i] == 5)||(conv_tableEM[devid][i] == 6)||(conv_tableEM[devid][i] == 7)
     ||(conv_tableEM[devid][i] == 8)){
      fprintf(stderr,"SING_EM_alloc: current version does not support conversion=%d\n",conv_tableEM[devid][i]);
    }

  }

  if(wordhalf == 1) nwordEM[devid]++;
  //  printf("EM length %d nwordEM %d\n",length,nwordEM[devid]);

  areaid[devid]++;  
  return (areaid[devid]-1);

}

static int nidp[NHIBMAX];
static UINT64 idpem[NHIBMAX][10000][4];
static UINT64 idpem1[NHIBMAX][10000][4];

void SING_EM_writeMC(int devid, int areaid, int start, int ndata, void *hostp)
// available only when ndata = nbb(16), (start%nbb)=0;
{
  int i,EMlin,EMcol,ii,jj,memadr,iii,iiii;
  int j;
  UINT64 tmp,tmp2;

  if(ndata != 16){
    fprintf(stderr,"SING_EM_write: ndata != 16\n");
    exit(1);	   
  }
  //  printf("sing_em_send: start %d ndata %d nwordEM %d\n",start,ndata,nwordEM[devid]);

  for(i=0;i<nconv_tableEM[devid];i++){
    tmp = *(((UINT64 *)hostp)+i);    
   //  printf("start %d EM %d %016llx\n",start,i,tmp);
  }

  idpem[devid][0][0] = idpem[devid][0][1] = idpem[devid][0][2] = 0x0;
  idpem1[devid][0][0] = idpem1[devid][0][1] = idpem1[devid][0][2] = 0x0;
  idpem[devid][0][3] = 0x0000003f00000000 |((UINT64)nwordEM[devid])<<56 ;
  idpem1[devid][0][3] = 0x1;

  iiii=0;
  for(j=0;j<16;j++){

    EMlin = ((start%16) * nwordEM[devid])/4 +1;
    EMcol = ((start%16) * nwordEM[devid])%4;
  //printf("EM nword start line column %d %d %d %d\n",nwordEM[devid],start,EMlin,EMcol);

    ii = EMlin;
    jj = EMcol;

    iii=0;
    for(i=0;i<nwordEM[devid];i++){

      if(conv_tableEM[devid][iii] == 0){
        idpem[devid][ii][jj] = *(((UINT64 *)hostp)+iiii); 
        iii++; iiii++;

      }else if(conv_tableEM[devid][iii] == 1){
        tmp = *(((UINT64 *)hostp)+iiii);
        if((iii+1)==nconv_tableEM[devid]){
  	  tmp2 = 0;
        }else{
          tmp2 = *(((UINT64 *)hostp)+iiii+1);	
        }
        idpem[devid][ii][jj] = (0xffffffff00000000 & tmp) | (0xfffffff & (tmp2 >> 36)) ;
        iii+=2; iiii+=2; 

      }else if(conv_tableEM[devid][iii] == 9){
        tmp = *(((UINT64 *)hostp)+iiii);
        idpem[devid][ii][jj] = (0xffffffff & tmp ) <<28 | (0xffffffff00000000 & tmp) >> 40 ;  // endian
        iii+=2; iiii++;

      }
      idpem1[devid][ii][jj] = 0x1;
      jj++;
      if(jj==4){
        jj = 0; ii++;
      }
    }

    start++;
  }

  nidp[devid] = 16*nwordEM[devid]/4+1;
  memadr = ((start-1)/16) * nidp[devid] +0x10000;
  em_write2(devid, nidp[devid], memadr);

}

int SING_EM_sendMC(int devid, int areaid, int start, int ndata, void *hostp)
  // areaid =0 , ndata = 1 (fixed);
  // assume arg. start is ordered
{
  int i,EMlin,EMcol,ii,jj,memadr,iii,iiii;
  UINT64 tmp,tmp2;
 
  //  printf("sing_em_send: start %d ndata %d nwordEM %d\n",start,ndata,nwordEM[devid]);

  for(i=0;i<nconv_tableEM[devid];i++){
    tmp = *(((UINT64 *)hostp)+i);    
   //  printf("start %d EM %d %016llx\n",start,i,tmp);
  }

  if((start%16)==0){
    idpem[devid][0][0] = idpem[devid][0][1] = idpem[devid][0][2] = 0x0;
    idpem1[devid][0][0] = idpem1[devid][0][1] = idpem1[devid][0][2] = 0x0;
    idpem[devid][0][3] = 0x0000003f00000000 |((UINT64)nwordEM[devid])<<56 ;
    idpem1[devid][0][3] = 0x1;
  }

  EMlin = ((start%16) * nwordEM[devid])/4 +1;
  EMcol = ((start%16) * nwordEM[devid])%4;
  //printf("EM nword start line column %d %d %d %d\n",nwordEM[devid],start,EMlin,EMcol);

  ii = EMlin;
  jj = EMcol;

  iii=iiii=0;
  for(i=0;i<nwordEM[devid];i++){

    if(conv_tableEM[devid][iii] == 0){
      idpem[devid][ii][jj] = *(((UINT64 *)hostp)+iiii); 
      iii++; iiii++;

    }else if(conv_tableEM[devid][iii] == 1){
      tmp = *(((UINT64 *)hostp)+iiii);
      if((iii+1)==nconv_tableEM[devid]){
	tmp2 = 0;
      }else{
        tmp2 = *(((UINT64 *)hostp)+iiii+1);	
      }
      idpem[devid][ii][jj] = (0xffffffff00000000 & tmp) | (0xfffffff & (tmp2 >> 36)) ;
      iii+=2; iiii+=2; 

    }else if(conv_tableEM[devid][iii] == 9){
      tmp = *(((UINT64 *)hostp)+iiii);
      idpem[devid][ii][jj] = (0xffffffff & tmp ) <<28 | (0xffffffff00000000 & tmp) >> 40 ;  // endian
      iii+=2; iiii++;

    }
    idpem1[devid][ii][jj] = 0x1;
    jj++;
    if(jj==4){
      jj = 0; ii++;
    }

  }

  if((start%16)==15){
    nidp[devid] = 16*nwordEM[devid]/4+1;
    memadr = (start/16) * nidp[devid] +0x10000;
    em_write2(devid, nidp[devid], memadr);

    for(i=0;i<nidp[devid];i++) idpem1[devid][i][0]= idpem1[devid][i][1]= idpem1[devid][i][2]= idpem1[devid][i][3]= 0;
    memadr = ((start/16)+1) * nidp[devid] +0x10000;
    em_write2(devid, nidp[devid], memadr);

  }

  return 0;
}

static void em_write2(int devid, int nidp, int adrem)
{
  int nn,ii;

  nn = 5*nidp;
  rbuf[devid][0] = 0x0000000700000000ll | nn;
  for(ii=0;ii<nidp;ii++){
    rbuf[devid][5*ii+1] = ((UINT64)(ii+adrem))<<32 | (0x1&idpem1[devid][ii][0])<<24 | (0x1&idpem1[devid][ii][1])<<16
      | (0x1&idpem1[devid][ii][2])<<8 | (0x1&idpem1[devid][ii][3]);
    rbuf[devid][5*ii+2] = idpem[devid][ii][0];
    rbuf[devid][5*ii+3] = idpem[devid][ii][1];
    rbuf[devid][5*ii+4] = idpem[devid][ii][2];
    rbuf[devid][5*ii+5] = idpem[devid][ii][3];
  }
  //for(ii=0;ii<nn+1;ii++) printf("em_write2 ii %x %d rbuf 0x%016llx\n",ii,ii,rbuf[devid][ii]);
  gdrtb_sendMC(devid, nn+1, rbuf[devid]);

}

//


static int ispid[NHIBMAX]={0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
static int ibegin0[NHIBMAX]={1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000};
static int imemadr[NHIBMAX]={0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};

#define ISPIDMAX 10

static UINT64 ispm[NHIBMAX][ISPIDMAX][1000];
static UINT64 ispl[NHIBMAX][ISPIDMAX][1000];
static int nisp[NHIBMAX][ISPIDMAX];
static int ibegin[NHIBMAX][ISPIDMAX];
static int ispidp[NHIBMAX][ISPIDMAX];
static int ispidppos[NHIBMAX][ISPIDMAX];
static int idpbegin[NHIBMAX][ISPIDMAX];
static int nrrn[NHIBMAX][ISPIDMAX];
static UINT64 rrndata[NHIBMAX][ISPIDMAX][100];
static int rrnpos[NHIBMAX][ISPIDMAX][100];
static int rrnconv[NHIBMAX][ISPIDMAX][100];
static int rrnmemadr[NHIBMAX][ISPIDMAX][100];
static int rrnn[NHIBMAX][ISPIDMAX][100];

void SING_iflushMC(int devid)
{
  ispid[devid] = 0;
  ibegin0[devid]=1000;
  imemadr[devid]=0;
}
     
static void run_counter(int devid, int nisp,int adrisp,int nrepisp,
                        int ncntidp,int nidp,int adridp, int memadr,int nrepidp)
{
  int i,id,size;

  rbuf[devid][0] = 0x0000000e00000002ll | ((UINT64)nisp)<<48 | adrisp<<16;
  rbuf[devid][1] = 0x0000000000000000ll | ((UINT64)nrepisp)<<40 | ((UINT64)ncntidp)<<24 | nrepidp;
  rbuf[devid][2] = 0x0000000000000000ll | ((UINT64)adridp)<<48 | ((UINT64)nidp)<<32 | memadr ;

  gdrtb_sendMC(devid, 3, rbuf[devid]);

}

void SING_executeMC(int devid, int init_id, int loop_id, int final_id, int loop_count, 
                 void *loop_out, void *final_out)
{
  int i,n,ibgn,memadr,ii,iii;

  //  printf("nisp %d %d %d\n",nisp[devid][init_id],nisp[devid][loop_id],nisp[devid][final_id]);
  //  printf("ibegin %d %d %d\n",ibegin[devid][init_id],ibegin[devid][loop_id],ibegin[devid][final_id]);  

  //  for(i=0;i<nisp[devid][init_id];i++) printf("isp %d %016llx %016llx\n",init_id,ispm[devid][init_id][i],ispl[devid][init_id][i]);
 //  for(i=0;i<nisp[devid][loop_id];i++) printf("isp %d %016llx %016llx\n",loop_id,ispm[devid][loop_id][i],ispl[devid][loop_id][i]);
  //  for(i=0;i<nisp[devid][final_id];i++) printf("isp %d %016llx %016llx\n",final_id,ispm[devid][final_id][i],ispl[devid][final_id][i]);  
  //    printf("ispidp %d %d %d\n",ispidp[devid][init_id],ispidp[devid][loop_id],ispidp[devid][final_id]);
  //    printf("nrrn %d %d %d\n",nrrn[devid][init_id],nrrn[devid][loop_id],nrrn[devid][final_id]);
  //    for(i=0;i<nrrn[devid][init_id];i++) printf("rrn %d %d %d %016llx %d %x\n",init_id,i,rrnpos[devid][init_id][i],rrndata[devid][init_id][i],rrnconv[devid][init_id][i],rrnmemadr[devid][init_id][i]);
  //    for(i=0;i<nrrn[devid][loop_id];i++) printf("rrn %d %d %d %016llx %d %x\n",init_id,i,rrnpos[devid][loop_id][i],rrndata[devid][loop_id][i],rrnconv[devid][loop_id][i],rrnmemadr[devid][loop_id][i]);
  //    for(i=0;i<nrrn[devid][final_id];i++) printf("rrn %d %d %d %016llx %d %x %d\n",init_id,i,rrnpos[devid][final_id][i],rrndata[devid][final_id][i],rrnconv[devid][final_id][i],rrnmemadr[devid][final_id][i],rrnn[devid][final_id][i]);

  //  nidp[devid] = 0;  
  //  printf("nidp %d\n",nidp[devid]);

  // run init_id
  n = nisp[devid][init_id]; ibgn = ibegin[devid][init_id];
  //  printf("init_id n %d %x ibgn %x\n",n,n,ibgn);
  run_counter(devid,2*n,2*ibgn,1,0,0,0,0,0);                  
  if(ispidp[devid][init_id]==1){
    run_counter(devid,0,0,0,20*nidp[devid],nidp[devid],0,0x10000,1);
  }

  // run loop_id
  n = nisp[devid][loop_id]; ibgn= ibegin[devid][loop_id];
  //  printf("loop_id n %d %x ibgn %x loop_count %d\n",n,n,ibgn,loop_count);
  if(ispidp[devid][loop_id]==1){
    run_counter(devid,2*n,2*ibgn,loop_count,2*n,nidp[devid],2*idpbegin[devid][loop_id],nidp[devid]+0x10000,loop_count);
    //    for(i=0;i<loop_count;i++){
    //      run_counter(devid,2*n,2*ibgn,1,2*n,nidp[devid],0,nidp[devid]*(i+1)+0x10000,1);     
    //    }
  }else{
    run_counter(devid,2*n,2*ibgn,loop_count,0,0,0,0,0);   
  }

  // run final_id
  n = rrnpos[devid][final_id][0]; ibgn = ibegin[devid][final_id];
  //  printf("final_id n %d %x ibgn %x\n",n,n,ibgn);
  run_counter(devid,2*n,2*ibgn,1,0,0,0,0,0);        

  iii = 0;
  for(i=0;i<nrrn[devid][final_id];i++){
    if(i==(nrrn[devid][final_id]-1)){
      n = nisp[devid][final_id]-rrnpos[devid][final_id][i];
    }else{
      n = rrnpos[devid][final_id][i+1]-rrnpos[devid][final_id][i];
    }
    ibgn = ibegin[devid][final_id] + rrnpos[devid][final_id][i]; 
    memadr = rrnmemadr[devid][final_id][i];

    //    printf("final_n %d %x ibgn %x memadr %x\n",n,n,ibgn,memadr);
    //    run_counter(devid,0,0,0,2*n,1,0,memadr,1);   (need solution)
    run_counter(devid,2*n,2*ibgn,1,2*n,1,0,memadr,1);

    hib_recvMC(devid, 2*rrnn[devid][final_id][i], wbuf[devid]);    

    for(ii=0;ii<(2*rrnn[devid][final_id][i]);ii+=4){
      //      printf("%d %d %d wbuf %016llx %016llx %016llx %016llx\n",final_id,i,ii,
      //	     wbuf[devid][ii],wbuf[devid][ii+1],wbuf[devid][ii+2],wbuf[devid][ii+3]);
    }

    //    run_counter(devid,2*n,2*ibgn,1,0,0,0,0,0);  (need solution)

    if(rrnconv[devid][final_id][i]==4){    
      for(ii=0;ii<rrnn[devid][final_id][i];ii++){
        *(((UINT64 *)final_out)+2*ii  +iii) = (*(wbuf[devid]+2*ii+1))>>28 ;
        *(((UINT64 *)final_out)+2*ii+1+iii) = (0xfffffff&(*(wbuf[devid]+2*ii+1)))<<8 | (*(wbuf[devid]+2*ii))>>56 ;	
      }
      iii += 2*rrnn[devid][final_id][i];
    }else{
      for(ii=0;ii<rrnn[devid][final_id][i];ii++){
        double dtmp;
        dtmp = *((double *)(wbuf[devid]+2*ii+1)) ;
        *(double *)(((UINT64 *)final_out)+ii+iii) = dtmp;
      }
      iii += rrnn[devid][final_id][i];
    }

    if((rrnconv[devid][final_id][i]!=0)&&(rrnconv[devid][final_id][i]!=4)){
      fprintf(stderr,"SING_execute: current version does not support rrnconv=%d\n",rrnconv[devid][final_id][i]);
    }
  }
}

int SING_iregisterMC(int devid, int ninsts, char **insts)
{
  int i,irrn;
  char cdata[100],*p,*p2;
  char cdata1[100];
  char cdata2[100];    
  UINT64 d[100];
  int nidp,memadr,idpdelay,refresh_delay,nplus,nplus0;
      
  // printf("%d\n",ninsts);
  nisp[devid][ispid[devid]] = ninsts;
  ispidp[devid][ispid[devid]] = 0;

  irrn=0;
  for(i=0;i<ninsts;i++){
    //    printf("%s\n",insts[i]);

    // isp 

    sscanf(insts[i],"%s%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x",
           cdata,&d[0],&d[1],&d[2],&d[3],&d[4],&d[5],&d[6],&d[7],&d[8],&d[9],&d[10],&d[11],&d[12],&d[13],&d[14],&d[15],&d[16],&d[17],&d[18],
	   &d[19],&d[20],&d[21],&d[22],&d[23],&d[24],&d[25],&d[26],&d[27],&d[28],&d[29],&d[30],&d[31],&d[32],&d[33],&d[34],&d[35],&d[36],
	   &d[37],&d[38],&d[39],&d[40],&d[41],&d[42],&d[43],&d[44],&d[45],&d[46],&d[47],&d[48],&d[49]);

    ispm[devid][ispid[devid]][i] = 0;
    ispm[devid][ispid[devid]][i] |= ((0x7 & d[0]) << 61) | ((0x7 & d[1]) << 58) | ((0x7 & d[2]) << 55) | ((0x1 & d[3]) << 54) | ((0x1 & d[4]) << 53);
    ispm[devid][ispid[devid]][i] |= ((0x1 & d[5]) << 52) | ((0x3 & d[6]) << 50) | ((0x1 & d[7]) << 49) | ((0x3 & d[8]) << 47) | ((0x3f & d[9]) << 41);
    ispm[devid][ispid[devid]][i] |= ((0x1 & d[10]) << 40) | ((0x1 & d[11]) << 39) | ((0x3f & d[12]) << 33) | ((0x1 & d[13]) << 32) | ((0x1 & d[14]) << 31);
    ispm[devid][ispid[devid]][i] |= ((0x3f & d[15]) << 25) | ((0x1 & d[16]) << 24) | ((0x1 & d[17]) << 23) | ((0x1 & d[18]) << 22) | ((0x3 & d[19]) << 20);
    ispm[devid][ispid[devid]][i] |= ((0x1ff & d[20]) << 11) | ((0xf & d[21]) << 7) | ((0x1 & d[22]) << 6) | ((0x1 & d[23]) << 5) | ((0x1 & d[24]) << 4);
    ispm[devid][ispid[devid]][i] |= ((0x1 & d[25]) << 3) | ((0x1 & d[26]) << 2) | ((0x1 & d[27]) << 1) | ((0x1 & d[28]) << 0);

    ispl[devid][ispid[devid]][i] = 0;
    ispl[devid][ispid[devid]][i] |= ((0x1 & d[29]) << 63) | ((0x1 & d[30]) << 62) | ((0x1 & d[31]) << 61) | ((0x1 & d[32]) << 60) | ((0x1 & d[33]) << 59);
    ispl[devid][ispid[devid]][i] |= ((0x3 & d[34]) << 57) | ((0x3 & d[35]) << 55) | ((0x1 & d[36]) << 54) | ((0x1 & d[37]) << 53) | ((0x1 & d[38]) << 52);
    ispl[devid][ispid[devid]][i] |= ((0x1 & d[39]) << 51) | ((0x1 & d[40]) << 50) | ((0x7 & d[41]) << 47) | ((0x3 & d[42]) << 45) | ((0x1f & d[43]) << 40);
    ispl[devid][ispid[devid]][i] |= ((0x1 & d[44]) << 39) | ((0x1 & d[45]) << 38) | ((0x1 & d[46]) << 37) | ((0x7ff & d[47]) << 26) | ((0x1f & d[48]) << 21);
    ispl[devid][ispid[devid]][i] |= ((0x1 & d[49]) << 20);

    //    printf("isp %016llx %016llx\n",ispm[devid][ispid[devid]][i],ispl[devid][ispid[devid]][i]);

    /*
    printf(" LOOP       :%x\n",d[0]);
    printf("M-Reg\n");  
    printf(" IMR(2:0)   :%x\n",d[1]); printf(" OMR(2:0)   :%x\n",d[2]); printf(" IFSEL      :%x\n",d[3]);
    printf("T-Reg\n");
    printf(" WRITE      :%x\n",d[4]); printf(" SHIRTSTOP  :%x\n",d[5]); printf(" ISEL(1:0)  :%x\n",d[6]);
    printf("Reg File\n");
    printf(" WRITE      :%x\n",d[7]); printf(" ISEL(1:0)  :%x\n",d[8]);  printf(" WADR(5:0)  :%x\n",d[9]);
    printf(" WADRI      :%x\n",d[10]); printf(" WWL        :%x\n",d[11]);  printf(" RADRA(5:0) :%x\n",d[12]);
    printf(" RADRIA     :%x\n",d[13]); printf(" RWLA       :%x\n",d[14]);  printf(" RADRB(5:0) :%x\n",d[15]);
    printf(" RADRIB     :%x\n",d[16]); printf(" RWLB       :%x\n",d[17]);  
    printf("LM\n");
    printf(" WRITE      :%x\n",d[18]); printf(" ISEL(1:0)  :%x\n",d[19]);  printf(" ADR(8:0)   :%x\n",d[20]);
    printf(" ADRI(3:0)  :%x\n",d[21]); printf(" TREGADR    :%x\n",d[22]);  printf(" WL         :%x\n",d[23]);
    printf("FMUL\n");
    printf(" SHIFT50A   :%x\n",d[24]); printf(" ROUND50A   :%x\n",d[25]);  printf(" ROUNDA     :%x\n",d[26]);
    printf(" NORMALA    :%x\n",d[27]); printf(" SHIFT25B   :%x\n",d[28]);  printf(" SHIFT50B   :%x\n",d[29]);
    printf(" ROUNDB     :%x\n",d[30]); printf(" NORMALB    :%x\n",d[31]);  printf(" ROUND      :%x\n",d[32]);
    printf(" NORMALO    :%x\n",d[33]); printf(" ISELA(1:0) :%x\n",d[34]);  printf(" ISELB(1:0) :%x\n",d[35]);
    printf("FADDSUB\n");
    printf(" NORMALA    :%x\n",d[36]); printf(" NORMALB    :%x\n",d[37]);  printf(" SIGNB      :%x\n",d[38]);
    printf(" ROUND      :%x\n",d[39]); printf(" NORMALO    :%x\n",d[40]);  printf(" ISELA(2:0) :%x\n",d[41]);
    printf(" ISELB(1:0) :%x\n",d[42]);
    printf("IALU\n");
    printf(" IALUOP(4:0):%x\n",d[43]); printf(" UNSIGNED   :%x\n",d[44]);
    printf("FADDSUB/IALU\n");
    printf(" FSEL       :%x\n",d[45]);
    printf("BM\n");
    printf(" WRITE      :%x\n",d[46]); printf(" ADR(10:0)  :%x\n",d[47]); printf(" PEADR(4:0) :%x\n",d[48]);
    printf(" WL         :%x\n",d[49]);
    */

    // idp

    if(strstr(insts[i],"idp")!=NULL){
      ispidp[devid][ispid[devid]] = 1;
      ispidppos[devid][ispid[devid]] = i;
    }

    // rrn

    if(strstr(insts[i],"rrn")!=NULL){
      p = strstr(insts[i],"rrn");
      //      printf("%s\n",p);
      sscanf(p,"%s%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x",cdata,&d[51],&d[52],&d[53],&d[54],&d[55],&d[56],&d[57],&d[58],
	      &d[59],&d[60],&d[61],&d[62],&d[63],&d[64],&d[65]);

      rrnpos[devid][ispid[devid]][irrn] = i;
      rrndata[devid][ispid[devid]][irrn] = ((0x1fff & d[51])<<51)| ((0xff & d[52])<<43)| ((0x1f & d[53])<<38)| ((0x1 & d[54]) <<37) | ((0x1 & d[55]) <<36);
      rrndata[devid][ispid[devid]][irrn] |= ((0x1 & d[56])<<35) | ((0x1 & d[57])<<34)| ((0x1 & d[58])<<33)	| ((0x1 & d[59])<<32) | ((0x1 & d[60])<<31);
      rrndata[devid][ispid[devid]][irrn] |= ((0x1 & d[61])<<30) | ((0x1f & d[62])<<25) | ((0x1 & d[63])<<24) | ((0x1 & d[64])<<23) | ((0x1 & d[65])<<22);
      
      /*
      printf("rrndata %llx\n",rrndata[devid][ispid[devid]][irrn]);
      printf("ADDR(12:0) : %4x\n",d[51]); printf("N(7:0)     : %4x %d\n",d[52],d[52]); printf("BBADR(4:0) : %4x\n",d[53]);
      printf("REDUC      : %4x\n",d[54]); printf("WL         : %4x\n",d[55]); printf("FSEL       : %4x\n",d[56]);
      printf("NORMALA    : %4x\n",d[57]); printf("NORMALB    : %4x\n",d[58]); printf("SIGNB      : %4x\n",d[59]);
      printf("ROUND      : %4x\n",d[60]); printf("NORMALO    : %4x\n",d[61]); printf("IALUOP(4:0): %4x\n",d[62]);
      printf("UNSIGNED   : %4x\n",d[63]); printf("ODPOE      : %4x\n",d[64]); printf("SREGEN     : %4x\n",d[65]);
      */

      rrnn[devid][ispid[devid]][irrn] = d[52];
      p2 = strstr(p,"conv");
      sscanf(p2,"%s%x",cdata,&d[66]);
      rrnconv[devid][ispid[devid]][irrn] = d[66];
      //      printf("conv %x\n",rrnconv[devid][ispid[devid]][irrn]);

      idp00[devid][0] = 0x0000000000000000; idp01[devid][0] = 0x0; 
      idp10[devid][0] = 0x0000000000000000; idp11[devid][0] = 0x0;
      idp00[devid][1] = 0x0180000000000000; idp01[devid][1] = 0x1; 
      idp10[devid][1] = rrndata[devid][ispid[devid]][irrn]; idp11[devid][1] = 0x1;
      nidp = 1; memadr = 0x700000 + imemadr[devid];
      em_write(devid, nidp, memadr);
      rrnmemadr[devid][ispid[devid]][irrn] = memadr;
      imemadr[devid]++; 
      irrn++;
    }
  }
  nrrn[devid][ispid[devid]] = irrn;

  // nop insersion

  if(Model[devid]==460){
    idpdelay = 0; 
  }else{
    idpdelay = 23; 
  }
  refresh_delay = 29 ; 
  if(ispidp[devid][ispid[devid]]==1){
    if(ispidppos[devid][ispid[devid]]<idpdelay){
      idpbegin[devid][ispid[devid]] = 0;
    }else{
      idpbegin[devid][ispid[devid]] = ispidppos[devid][ispid[devid]] - idpdelay + 2  ;
    }
    nplus0 = (idpbegin[devid][ispid[devid]] + idpdelay + nwordEM[devid]/4*16 + refresh_delay ) - nisp[devid][ispid[devid]];
    if(nplus0<0){
      nplus = 0;
    }else{
      nplus = nplus0;
    }
    printf("%d nisp %d idppos %d idpbegin %d em->bm %d nplus %d %d\n",
	   ispid[devid],nisp[devid][ispid[devid]],ispidppos[devid][ispid[devid]],idpbegin[devid][ispid[devid]],
	   nwordEM[devid]/4*16,nplus0,nplus+nisp[devid][ispid[devid]]);
    for(i=0;i<nplus;i++){
      ispm[devid][ispid[devid]][nisp[devid][ispid[devid]]+i] = 0;
      ispl[devid][ispid[devid]][nisp[devid][ispid[devid]]+i] = 0;
    }      
    nisp[devid][ispid[devid]] += nplus;
  }

  ibegin[devid][ispid[devid]] = ibegin0[devid];
  send_ispdata(devid,nisp[devid][ispid[devid]],ibegin[devid][ispid[devid]],ispm[devid][ispid[devid]],ispl[devid][ispid[devid]]);
  ibegin0[devid] += nisp[devid][ispid[devid]];

  //  printf("ispid %d\n",ispid[devid]);
  ispid[devid]++;  
  return (ispid[devid]-1);
}

static void em_write(int devid, int nidp, int adrem)
{
  int nn,ii;

  nn = 5*nidp;
  rbuf[devid][0] = 0x0000000700000000ll | nn;
  for(ii=0;ii<nidp;ii++){
    rbuf[devid][5*ii+1] = ((UINT64)(ii+adrem))<<32 | (0x1&idp01[devid][2*ii])<<24 | (0x1&idp11[devid][2*ii])<<16
      | (0x1&idp01[devid][2*ii+1])<<8 | (0x1&idp11[devid][2*ii+1]);
    rbuf[devid][5*ii+2] = idp00[devid][2*ii];
    rbuf[devid][5*ii+3] = idp10[devid][2*ii];
    rbuf[devid][5*ii+4] = idp00[devid][2*ii+1];
    rbuf[devid][5*ii+5] = idp10[devid][2*ii+1];
  }
  //for(ii=0;ii<nn+1;ii++) printf("idp00 ii %x %d rbuf 0x%016llx\n",ii,ii,rbuf[devid][ii]);
  gdrtb_sendMC(devid, nn+1, rbuf[devid]);
}

static void send_ispdata(int devid, int n, int ibegin, UINT64 ispmdata[],UINT64 ispldata[])
{
  int i;

  //  printf("devid n ibegin %d %d %d\n",devid,n,ibegin);

  rbuf[devid][0] = 0x0000000600000000ll | (2*n) | (2*ibegin)<<16 ;
  for(i=0;i<n;i++){
    rbuf[devid][2*i+1] = ispmdata[i];
    rbuf[devid][2*i+2] = ispldata[i];
    //    printf("i %d rbuf ispm ispl %llx %llx %llx %llx\n",i,rbuf[devid][2*i+1],rbuf[devid][2*i+2],ispmdata[i],ispldata[i]);
  }
  gdrtb_sendMC(devid, (2*n+1), rbuf[devid]);

}

//#define TB3x8
static void
gdrtb_sendMC(int devid, int size, UINT64 *buf)
{
#ifdef TB3x8
  int i;

  for(i=(size-1);i>=0;i--){
    buf[2*i] = buf[i];
    buf[2*i+1] = 0;
  }
  //  for(i=0;i<2*size;i++) printf("i %d buf %llx\n",i,buf[i]);

  hib_sendMC(devid, 2*size, buf);
#else
  hib_sendMC(devid, size, buf);
#endif
}

static void send_vector(int devid, int i0, int i1, UINT64 idp0data[], UINT64 idp1data[])
{
  int i,nn,data,n;
  int par[14],j;

  n= i1-i0 +1;
  //  printf("n; %d\n", n);
  if((n%2)==1){
    idp0data[i1+1] = 0; idp1data[i1+1] = 0;    
    n++;
  }
  // for(i=0;i<n;i++) printf("0x%016llx %5x\n",idp0data[i],idp1data[i]);
  n /= 2;
  //  printf("n; %d\n", n);

     /* ISP */
     rbuf[devid][0] = 0x0000000400000000ll | n ;
     for(i=0;i<n;i++) rbuf[devid][i+1] = 0x0;
     gdrtb_sendMC(devid, n+1, rbuf[devid]);

     /* IDP00 */
     rbuf[devid][0] = 0x0000000000000000ll | n ;
     for(i=0;i<n;i++) rbuf[devid][i+1] = idp0data[i0+2*i];
     gdrtb_sendMC(devid, n+1, rbuf[devid]);

     /* IDP01 */
     rbuf[devid][0] = 0x0000000100000000ll | n ;
     for(i=0;i<n;i++) rbuf[devid][i+1] = idp1data[i0+2*i];
     gdrtb_sendMC(devid, n+1, rbuf[devid]);

     /* IDP10 */
     rbuf[devid][0] = 0x0000000200000000ll | n ;
     for(i=0;i<n;i++) rbuf[devid][i+1] = idp0data[i0+2*i+1];
     gdrtb_sendMC(devid, n+1, rbuf[devid]);

     /* IDP11 */
     rbuf[devid][0] = 0x0000000300000000ll | n ;
     for(i=0;i<n;i++) rbuf[devid][i+1] = idp1data[i0+2*i+1];
     gdrtb_sendMC(devid, n+1, rbuf[devid]);

     /* RUN */
     nn=1;
     rbuf[devid][0] = 0x0000000800000000ll | nn;
     for(i=1;i<nn+1;i++) rbuf[devid][i] = 0x0000000000000000ll;
     gdrtb_sendMC(devid, nn+1, rbuf[devid]);

}

/*
 * note that nword & conv_table must be set
 * by SING_LM_write_regconvMC() in advance.
 */
static int sizeof_i_particle_vector_struct(void)
{
    int i, ii = 0, iii = 0;
    static int size = 0;
    int devid = Devid[0];

    // below may not work.
    // if(size != 0){
    //     return size; // returns a cached value.
    // }

    for(i=0;i<nword[devid];i++){
        if(conv_table[devid][iii]==0){
            iii++; ii++;
        }else if(conv_table[devid][iii]==1){
            iii+=2; ii+=2; 
        }else if(conv_table[devid][iii]==9){
            iii+=2; ii++;
        }
    }
    size = ii * sizeof(UINT64);
    return size;
}

static int sizeof_final_out(int final_id)
{
    int i, iii = 0;
    int devid = Devid[0];

    for(i=0;i<nrrn[devid][final_id];i++){
        if(rrnconv[devid][final_id][i]==4){    
            iii += 2*rrnn[devid][final_id][i];
        }else{
            iii += rrnn[devid][final_id][i];
        }
    }
    return iii * sizeof(UINT64);
}
