Skip to content

Commit

Permalink
powerpc/pseries/mm: call H_BLOCK_REMOVE
Browse files Browse the repository at this point in the history 
This hypervisor's call allows to remove up to 8 ptes with only call to
tlbie.

The virtual pages must be all within the same naturally aligned 8 pages
virtual address block and have the same page and segment size encodings.

Cc: "Aneesh Kumar K.V" <[email protected]>
Cc: Nicholas Piggin <[email protected]>
Cc: Michael Ellerman <[email protected]>
Cc: Paul Mackerras <[email protected]>
Cc: Benjamin Herrenschmidt <[email protected]>
Signed-off-by: Laurent Dufour <[email protected]>
Signed-off-by: Michael Ellerman <[email protected]>
  • Loading branch information
@mpe
Laurent Dufour authored and mpe committed Sep 17, 2018
1 parent 0effa48 commit ba2dd8a
Show file tree
Hide file tree
Showing 2 changed files with 207 additions and 8 deletions.
1 change: 1 addition & 0 deletions arch/powerpc/include/asm/hvcall.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -278,6 +278,7 @@
#define H_COP 0x304
#define H_GET_MPP_X 0x314
#define H_SET_MODE 0x31C
#define H_BLOCK_REMOVE 0x328
#define H_CLEAR_HPT 0x358
#define H_REQUEST_VMC 0x360
#define H_RESIZE_HPT_PREPARE 0x36C
Expand Down
214 changes: 206 additions & 8 deletions arch/powerpc/platforms/pseries/lpar.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -417,24 +417,137 @@ static void pSeries_lpar_hpte_invalidate(unsigned long slot, unsigned long vpn,
BUG_ON(lpar_rc != H_SUCCESS);
}


/*
* As defined in the PAPR's section 14.5.4.1.8
* The control mask doesn't include the returned reference and change bit from
* the processed PTE.
*/
#define HBLKR_AVPN 0x0100000000000000UL
#define HBLKR_CTRL_MASK 0xf800000000000000UL
#define HBLKR_CTRL_SUCCESS 0x8000000000000000UL
#define HBLKR_CTRL_ERRNOTFOUND 0x8800000000000000UL
#define HBLKR_CTRL_ERRBUSY 0xa000000000000000UL

/**
* H_BLOCK_REMOVE caller.
* @idx should point to the latest @param entry set with a PTEX.
* If PTE cannot be processed because another CPUs has already locked that
* group, those entries are put back in @param starting at index 1.
* If entries has to be retried and @retry_busy is set to true, these entries
* are retried until success. If @retry_busy is set to false, the returned
* is the number of entries yet to process.
*/
static unsigned long call_block_remove(unsigned long idx, unsigned long *param,
bool retry_busy)
{
unsigned long i, rc, new_idx;
unsigned long retbuf[PLPAR_HCALL9_BUFSIZE];

if (idx < 2) {
pr_warn("Unexpected empty call to H_BLOCK_REMOVE");
return 0;
}
again:
new_idx = 0;
if (idx > PLPAR_HCALL9_BUFSIZE) {
pr_err("Too many PTEs (%lu) for H_BLOCK_REMOVE", idx);
idx = PLPAR_HCALL9_BUFSIZE;
} else if (idx < PLPAR_HCALL9_BUFSIZE)
param[idx] = HBR_END;

rc = plpar_hcall9(H_BLOCK_REMOVE, retbuf,
param[0], /* AVA */
param[1], param[2], param[3], param[4], /* TS0-7 */
param[5], param[6], param[7], param[8]);
if (rc == H_SUCCESS)
return 0;

BUG_ON(rc != H_PARTIAL);

/* Check that the unprocessed entries were 'not found' or 'busy' */
for (i = 0; i < idx-1; i++) {
unsigned long ctrl = retbuf[i] & HBLKR_CTRL_MASK;

if (ctrl == HBLKR_CTRL_ERRBUSY) {
param[++new_idx] = param[i+1];
continue;
}

BUG_ON(ctrl != HBLKR_CTRL_SUCCESS
&& ctrl != HBLKR_CTRL_ERRNOTFOUND);
}

/*
* If there were entries found busy, retry these entries if requested,
* of if all the entries have to be retried.
*/
if (new_idx && (retry_busy || new_idx == (PLPAR_HCALL9_BUFSIZE-1))) {
idx = new_idx + 1;
goto again;
}

return new_idx;
}

#ifdef CONFIG_TRANSPARENT_HUGEPAGE
/*
* Limit iterations holding pSeries_lpar_tlbie_lock to 3. We also need
* to make sure that we avoid bouncing the hypervisor tlbie lock.
*/
#define PPC64_HUGE_HPTE_BATCH 12

static void __pSeries_lpar_hugepage_invalidate(unsigned long *slot,
unsigned long *vpn, int count,
int psize, int ssize)
static void hugepage_block_invalidate(unsigned long *slot, unsigned long *vpn,
int count, int psize, int ssize)
{
unsigned long param[PLPAR_HCALL9_BUFSIZE];
int i = 0, pix = 0, rc;
unsigned long flags = 0;
int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
unsigned long shift, current_vpgb, vpgb;
int i, pix = 0;

if (lock_tlbie)
spin_lock_irqsave(&pSeries_lpar_tlbie_lock, flags);
shift = mmu_psize_defs[psize].shift;

for (i = 0; i < count; i++) {
/*
* Shifting 3 bits more on the right to get a
* 8 pages aligned virtual addresse.
*/
vpgb = (vpn[i] >> (shift - VPN_SHIFT + 3));
if (!pix || vpgb != current_vpgb) {
/*
* Need to start a new 8 pages block, flush
* the current one if needed.
*/
if (pix)
(void)call_block_remove(pix, param, true);
current_vpgb = vpgb;
param[0] = hpte_encode_avpn(vpn[i], psize, ssize);
pix = 1;
}

param[pix++] = HBR_REQUEST | HBLKR_AVPN | slot[i];
if (pix == PLPAR_HCALL9_BUFSIZE) {
pix = call_block_remove(pix, param, false);
/*
* pix = 0 means that all the entries were
* removed, we can start a new block.
* Otherwise, this means that there are entries
* to retry, and pix points to latest one, so
* we should increment it and try to continue
* the same block.
*/
if (pix)
pix++;
}
}
if (pix)
(void)call_block_remove(pix, param, true);
}

static void hugepage_bulk_invalidate(unsigned long *slot, unsigned long *vpn,
int count, int psize, int ssize)
{
unsigned long param[PLPAR_HCALL9_BUFSIZE];
int i = 0, pix = 0, rc;

for (i = 0; i < count; i++) {

Expand Down Expand Up @@ -462,6 +575,23 @@ static void __pSeries_lpar_hugepage_invalidate(unsigned long *slot,
param[6], param[7]);
BUG_ON(rc != H_SUCCESS);
}
}

static inline void __pSeries_lpar_hugepage_invalidate(unsigned long *slot,
unsigned long *vpn,
int count, int psize,
int ssize)
{
unsigned long flags = 0;
int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);

if (lock_tlbie)
spin_lock_irqsave(&pSeries_lpar_tlbie_lock, flags);

if (firmware_has_feature(FW_FEATURE_BLOCK_REMOVE))
hugepage_block_invalidate(slot, vpn, count, psize, ssize);
else
hugepage_bulk_invalidate(slot, vpn, count, psize, ssize);

if (lock_tlbie)
spin_unlock_irqrestore(&pSeries_lpar_tlbie_lock, flags);
Expand Down Expand Up @@ -564,6 +694,68 @@ static inline unsigned long compute_slot(real_pte_t pte,
return slot;
}

/**
* The hcall H_BLOCK_REMOVE implies that the virtual pages to processed are
* "all within the same naturally aligned 8 page virtual address block".
*/
static void do_block_remove(unsigned long number, struct ppc64_tlb_batch *batch,
unsigned long *param)
{
unsigned long vpn;
unsigned long i, pix = 0;
unsigned long index, shift, slot, current_vpgb, vpgb;
real_pte_t pte;
int psize, ssize;

psize = batch->psize;
ssize = batch->ssize;

for (i = 0; i < number; i++) {
vpn = batch->vpn[i];
pte = batch->pte[i];
pte_iterate_hashed_subpages(pte, psize, vpn, index, shift) {
/*
* Shifting 3 bits more on the right to get a
* 8 pages aligned virtual addresse.
*/
vpgb = (vpn >> (shift - VPN_SHIFT + 3));
if (!pix || vpgb != current_vpgb) {
/*
* Need to start a new 8 pages block, flush
* the current one if needed.
*/
if (pix)
(void)call_block_remove(pix, param,
true);
current_vpgb = vpgb;
param[0] = hpte_encode_avpn(vpn, psize,
ssize);
pix = 1;
}

slot = compute_slot(pte, vpn, index, shift, ssize);
param[pix++] = HBR_REQUEST | HBLKR_AVPN | slot;

if (pix == PLPAR_HCALL9_BUFSIZE) {
pix = call_block_remove(pix, param, false);
/*
* pix = 0 means that all the entries were
* removed, we can start a new block.
* Otherwise, this means that there are entries
* to retry, and pix points to latest one, so
* we should increment it and try to continue
* the same block.
*/
if (pix)
pix++;
}
} pte_iterate_hashed_end();
}

if (pix)
(void)call_block_remove(pix, param, true);
}

/*
* Take a spinlock around flushes to avoid bouncing the hypervisor tlbie
* lock.
Expand All @@ -583,6 +775,11 @@ static void pSeries_lpar_flush_hash_range(unsigned long number, int local)
if (lock_tlbie)
spin_lock_irqsave(&pSeries_lpar_tlbie_lock, flags);

if (firmware_has_feature(FW_FEATURE_BLOCK_REMOVE)) {
do_block_remove(number, batch, param);
goto out;
}

psize = batch->psize;
ssize = batch->ssize;
pix = 0;
Expand Down Expand Up @@ -621,6 +818,7 @@ static void pSeries_lpar_flush_hash_range(unsigned long number, int local)
BUG_ON(rc != H_SUCCESS);
}

out:
if (lock_tlbie)
spin_unlock_irqrestore(&pSeries_lpar_tlbie_lock, flags);
}
Expand Down

0 comments on commit ba2dd8a

Please sign in to comment.