From c4361581526bd00cd883a56d5ce8174f5f4b6106 Mon Sep 17 00:00:00 2001 From: Yuwen Hu Date: Fri, 16 Aug 2024 13:52:17 +0800 Subject: [PATCH 1/7] Update all-in-one benchmark prompts for continuation task --- .../prompt/continuation/longbench_2k.txt | 8 + .../prompt/continuation/longbench_4k.txt | 760 ++++++++++++++++++ python/llm/dev/benchmark/all-in-one/run.py | 13 +- 3 files changed, 775 insertions(+), 6 deletions(-) create mode 100644 python/llm/dev/benchmark/all-in-one/prompt/continuation/longbench_2k.txt create mode 100644 python/llm/dev/benchmark/all-in-one/prompt/continuation/longbench_4k.txt diff --git a/python/llm/dev/benchmark/all-in-one/prompt/continuation/longbench_2k.txt b/python/llm/dev/benchmark/all-in-one/prompt/continuation/longbench_2k.txt new file mode 100644 index 00000000000..ce70bcba3b6 --- /dev/null +++ b/python/llm/dev/benchmark/all-in-one/prompt/continuation/longbench_2k.txt @@ -0,0 +1,8 @@ +You are given a report by a government agency. Write a one-page summary of the report. + +Report: +Justice Management Division (JMD) JMD provides the Federal Bureau of Prisons senior management with guidance as it relates to Department of Justice (DOJ) policy for all matters pertaining to organization, management, and administration, including the use of human capital flexibilities such as retention incentives. BOP is responsible for incarcerating all federal offenders sentenced to prison. To carry out its mission, BOP, under the oversight of DOJ’s JMD, manages the human resource operations of its institutions, including the use of retention incentives. BOP administers, monitors, and oversees retention incentives through its Central Office, regional offices, and institutions. Central Office. The Central Office serves as BOP’s headquarters and provides oversight of BOP operations and program areas. Within the Central Office is BOP’s Human Resource Management Division (HRMD) which is responsible for developing, implementing and administering human resource policies and programs, including the use of retention incentives that meet OPM and DOJ requirements. In addition, the Central Office’s Program Review Division (PRD) is responsible for assessing BOP programs, including human resources, to ensure that they are managed and operated effectively. Regional offices. BOP has six regional offices that cover the Mid- Atlantic, North Central, Northeast, South Central, Southeast, and Western regions of the United States. These offices, each led by a regional director, oversee the operations of the 122 federal institutions within their respective geographic regions of the country. According to BOP officials, regional office staff also provide local level oversight of institutions’ human capital programs, such as retention incentives, among other things. Institutions. BOP institutions are managed by a warden and other officials, including an executive assistant and associate warden who generally provide overall direction and, in part, administer the institution’s human capital policies, including policies on retention incentives. Correctional services staff represent the largest segment of each institution’s workforce and are responsible for the correctional treatment, custody, and supervision of inmates. Non-correctional services staff include, among others, those employees assigned to non-correctional services management, facility operations, and the health services unit. Workers in health services and psychology services are responsible for providing inmates with medical, dental, and mental health services and include, for example, dentists, pharmacists, physicians, nurses, psychologists, and drug treatment specialists. The Federal Employees Pay Comparability Act of 1990 first authorized OPM to allow federal agencies to give incentives, including retention incentives, to employees. The Federal Workforce Flexibility Act of 2004 provided federal agencies increased flexibilities regarding these incentives. For example, individual retention incentives that were capped at 25 percent of an employee’s basic pay rate could be increased up to 50 percent in cases of critical agency need with OPM’s approval. Generally, under OPM regulations, an agency is authorized to pay a retention incentive to employees. This happens when the agency determines that the unusually high or unique qualifications of the employee or a special need of the agency for the employee’s services makes it essential to retain the employee and that the employee would be likely to leave federal service in the absence of an incentive. In addition, OPM requires agencies to develop plans for using retention incentives outlining, in part, the required documentation for justifying the retention incentive and any criteria for determining the amount of incentive and the length of the service period. Generally, agencies must require that employees sign a written service agreement that outlines the terms of the service such as the employee’s agreement to remain a certain length of time with the agency. Additionally, according to OPM regulations, to qualify for a retention incentive, each employee must have a performance rating of at least “fully successful” or an agency’s equivalent performance rating. BOP funds the majority of its retention incentives through its Salaries and Expenses appropriation account which represented almost 93 percent of BOP’s budget in FY 2016. According to BOP officials, BOP’s Central Office allocates funding from the Salaries and Expenses account to the regional offices. These regional offices then determine how to allocate their budget among various salary and expense activities, including retention incentives. HRMD delegates retention incentive determinations to each institution. In accordance with OPM requirements and BOP’s October 2016 Program Statement on Compensation, the wardens make retention incentive requests based on documented evidence that the employee possesses unusually high or unique qualifications or meets a special need of the agency and has a performance rating of at least “successful or its equivalent.” These incentives are calculated as a percentage of the employee’s basic pay and are disbursed in installments to the employee each pay period. In addition to retention incentives, BOP has authority to provide other compensation-based human capital flexibilities to employees, in certain circumstances. The following summarizes some of the compensation- based human capital flexibilities that BOP uses in addition to retention incentives, to retain and recruit staff: Recruitment and relocation incentives. BOP pays recruitment incentives to new hires and relocation incentives to current employees who elect to move to a different geographic area, when a position is likely to be difficult to fill in the absence of an incentive. Student loan repayments. Using this authority, BOP may repay federally-insured student loans to attract job candidates or retain current employees. Special salary rates. With OPM approval, BOP may establish higher rates of pay for an occupation or group of occupations nationwide or in a local area when it finds the government’s recruitment or retention efforts are, or would likely become, significantly handicapped without those higher rates. Physicians and dental comparability allowances. Comparability allowances may be paid to certain eligible physicians or dental professionals who enter into service agreements. These allowances are paid only to categories of physicians and dentists for which the agency is experiencing recruitment and retention problems and are fixed at the minimum amounts necessary to deal with such problems. BOP retention incentive expenditures generally increased from $10.7 million in fiscal year 2012 to $14.0 million in fiscal year 2016. Additionally, as illustrated in table 1, the number of employees who received retention incentives increased each year from 2,024 employees in fiscal year 2012 to 2,460 employees in fiscal year 2016. In general, BOP employees who received retention incentives received the incentive for more than one year. For example, from fiscal year 2012 through fiscal year 2016, a total of 3,382 BOP employees received retention incentive payments. Of those, 82 percent (2,766 of 3,382) received retention incentive payments for at least 2 years and 39 percent received retention incentives all 5 years, as shown in figure 1. From fiscal years 2012 through 2016, BOP spent more than 97 percent of its total retention incentive expenditures on employees at four California institutions and for medical professionals nationwide. BOP’s total retention incentive expenditures for the four California institutions and medical professionals nationwide in fiscal year 2016 are provided in figure 2. Four California Institutions. The California institutions—United States Penitentiary (USP) Atwater, Federal Correctional Institution (FCI) Herlong, FCI Mendota, and Federal Correctional Complex (FCC) Victorville—constituted the largest portion of BOP’s total retention incentive expenditures, and the level of their expenditures remained relatively steady from fiscal year 2012 through 2016. BOP provides group retention incentives for staff at the General Schedule (GS) grades level 12 and below and those in the Federal Wage System at three institutions—USP Atwater, FCI Herlong, and FCC Victorville. BOP also provides individual retention incentives to its employees at GS grades level 12 and below and in the Federal Wage System at FCI Mendota. As shown in figure 3, our analysis of BOP data found that from fiscal years 2012 through 2016, these four California institutions had the largest percentage of retention incentive expenditures across institutions as well as the largest percentage of employees who received retention incentives. Additionally, the four California institutions’ retention incentive expenditures remained relatively steady—around $8.1 to $8.2 million during the 5-year period—even though the overall number of employees who received the incentives generally increased. BOP officials told us that these California institutions’ retention incentive expenditures remained relatively steady in spite of an overall increase in the number of employees receiving incentives, in part, because in fiscal year 2013 BOP reduced the retention incentive rate—the percentage of an employee’s basic pay that determines the employee’s retention incentive— by 3 percent at the four California institutions. BOP officials reported using retention incentives primarily at these four institutions to supplement correctional officers’ salaries and compensate for the gap between BOP’s and other institutions’ salaries. Specifically, officials told us that these four California institutions were consistently understaffed as a result of their lower salaries in comparison to salaries offered at California state and local prisons and at other BOP institutions in California metropolitan areas. The Department of Labor’s Bureau of Labor Statistics reports that the average salary for correctional officers in California in 2016 was $70,020. For the same year, the annual average salary for BOP correctional officers at these four institutions was $50,859. To bring these four California institutions’ salaries in line with those offered by state, local, and other BOP institutions in California metropolitan areas, BOP officials told us that they first use recruitment incentives to attract and hire staff and then provide retention incentives to employees with a performance rating of at least “successful.” Medical Professionals. From fiscal years 2012 through 2016, BOP retention incentive expenditures for medical professionals increased by an average of approximately 21 percent per year. Our analysis showed that most recently—for fiscal years 2015 and 2016—BOP retention incentive expenditures for medical professionals accounted for the largest portion of BOP’s total retention incentive expenditures across the various occupation groups and was primarily responsible for the overall increase in BOP’s total retention incentive expenditures from fiscal year 2012 through fiscal year 2016. For example, in fiscal year 2016, BOP spent approximately 42 percent of total retention incentives expenditures for medical professionals ($5.8 million), 27 percent on correctional officers ($3.8 million), and the remaining 31 percent on employees in other occupations. In total, BOP retention incentive expenditures for medical professionals increased from approximately $2.7 million in fiscal year 2012 to $5.8 million in fiscal year 2016, as shown in figure 4. The increase accounted for 92 percent of BOP’s total increase in retention incentive expenditures during the five-year period. In comparison, BOP’s retention incentive expenditures for correctional officers and all other occupations remained relatively steady from fiscal year 2012 through fiscal year 2016, increasing by an average of approximately 1 percent per year. According to our analysis, the increase in retention incentive expenditures for medical professionals during the five years is partially explained by the increase in the number of institutions providing retention incentives to medical professionals. Specifically, from fiscal years 2012 through 2016, the number of institutions providing retention incentives to medical professionals increased from 53 institutions with 341 employees in medical occupations receiving retention incentives to 84 institutions providing retention incentives to a total of 646 employees in medical occupations. According to BOP officials, BOP primarily uses retention incentives for medical professionals in an effort to retain these employees by supplementing BOP salaries which are generally lower than salaries offered to medical professionals in the private sector. Officials told us that BOP has designated medical professional positions as hard-to-fill and, therefore, BOP retaining these professionals in a correctional setting requires the use of a variety of incentives, including retention incentives, in order to increase pay. BOP has a number of internal controls in place to ensure that retention incentive applications meet BOP and other requirements. BOP officials told us that these controls are part of a multilayered application and review process that begins at the institution and culminates at BOP’s Central Office. Our review of a random sample of 40 application packet case files for retention incentives awarded from fiscal year 2014 through fiscal year 2016 found that they all generally incorporated the internal controls described by officials. The key controls in this process include: Application review at the institution and regional levels. According to BOP officials, the retention incentive application process begins with an institution’s human resources office, whose staff complete a retention incentive application on behalf of an employee. The institution’s human resources office verifies that the information in the application justifies a retention incentive and that funds are available to pay the incentive. Although it is not required, BOP officials said that they use a retention incentive application checklist to help institutions ensure that retention incentive applications are complete. The institution’s human resources office then submits the completed application packet, which includes supporting documentation, to the warden for review. Next, the application packet is forwarded to the respective BOP regional director who also reviews it for accuracy and completeness. The regional director then adds an approval statement and forwards the packet to the Central Office for final review and approval. Of the 40 randomly selected application packet case files that we reviewed, 36 included a retention incentive checklist used by the institutions and all contained information to justify the retention incentive as well as a statement of the regional director’s approval. Central Office’s final application approval. BOP policy requires that all retention incentive applications undergo two levels of review in BOP’s Central Office: first by the Human Resource Management Division’s (HRMD) Staffing and Employee Relations Section (SERS) and next by HRMD’s Personnel Director, for final review and approval. According to BOP officials, during the review process there is ongoing communication between the various entities to ensure that applications are complete and accurate; for example, if SERS finds an error in the application or requests additional information, SERS returns the application to the regional or institutional level for correction and re-review. All of the 40 BOP application packet case files that we reviewed included approvals by HRMD’s Personnel Director or an authorized official, as required by BOP policy. Annual review and re-certification to continue retention incentives. According to BOP policy, on an annual basis, institutions’ human resources offices are required to review employees’ retention incentives to determine whether the incentive is still warranted. Payment of a retention incentive may be recertified and continued as long as the conditions giving rise to the original determination to pay the incentive still exist and funds are available. For each retention incentive, an institution’s human resources office must determine whether to continue, adjust, or terminate the incentive within one year of the initial or most recent approval. If the human resources office decides to continue the retention incentive, the institution’s warden must again submit a retention incentive application. Applications to continue the retention incentive proceed through the same review and approval process as initial applications. Of the 40 application files that we reviewed, 29 were continuations and 8 were initial requests for a retention incentive. According to BOP officials, after the initial approval of a retention incentive, an institution’s human resources office has primary responsibility for the monitoring of retention incentive payments. According to officials, institutions use a variety of internal controls to monitor the expiration, continuation, or termination of retention incentives, for example: Monitoring expiration dates. BOP officials stated that institutions’ human resources offices monitor retention incentives in order to identify incentives that are approaching their expiration date and need to be terminated or renewed. For example, according to BOP officials from USP Atwater, FCC Butner and FCI Phoenix, staff from their institutions’ human resources offices may generate a retention incentive activity report and cross reference this report with their locally generated tracking sheets. This process helps identify retention incentives approaching their expiration dates so that the human resources offices can submit a request for continuation before the incentive expires. Using automated reminders to prompt file review. BOP officials stated that institutions use automated reminders to alert human resources staff to check the records of retention incentive recipients for human resources-related events such as promotions or relocations that could affect the continuation of a retention incentive. Following a checklist of steps for relocation processes. BOP officials told us that in April 2016 they instituted a checklist that outlines steps that an institution’s human resources staff must take when employees relocate to a different institution. Based on our review of this checklist, one step on the sheet prompts human resources staff to review the employee’s retention incentive. According to BOP policy, when an employee receiving a retention incentive transfers to another location, the human resources office where the employee was receiving the retention incentive is responsible for submitting a request to terminate the incentive. The termination must be effective the last day of the pay period that the employee occupies the position. Submitting forgiveness waivers. BOP officials told us that institutions submit forgiveness waivers if a request to continue a retention incentive is not submitted and approved prior to the retention incentive expiring. BOP officials said that a forgiveness waiver is considered an acknowledgement of an administrative error and is a late submission of a retention incentive renewal that was still warranted. The waiver is not a request to forgive an overpayment since the employee was still considered to be eligible for the retention incentive. Of the 40 retention incentive applications that we reviewed, 5 applications included forgiveness waivers to excuse the tardiness of the filing and request continuations of the retention incentive. According to BOP officials, BOP conducts periodic audits and reviews of its human capital activities and related internal controls, to ensure that retention incentives are being used appropriately. The following offices conduct various audits and reviews involving BOP’s retention incentives: BOP’s Program Review Division (PRD) audits regional and institutional human resources functions. PRD audits BOP’s regional and institutional human resources offices to ensure that they are in compliance with BOP policies and procedures. According to BOP officials, as part of the audit process, PRD audits retention incentives to ensure that they have the proper approvals and are justified. PRD audits each institution’s human resources office at least every three years. During these audits, PRD generates retention incentive activity reports (the same reports that institutions run when monitoring for expiration dates), to check the accuracy of retention incentive programs under review. Following each audit, PRD issues a final report with findings to the institution and to the staff operating the program area under audit. Institutions respond to the report with corrective actions that the institution will take to address the findings. When the institution has resolved all corrective actions from the audit, the audit is closed. Additionally, each quarter, PRD provides HRMD with a report that summarizes its quarterly audit findings. According to BOP officials, HRMD uses these reports to identify any agency-wide trends that need to be addressed. Our review of BOP data showed that between fiscal years 2012 and 2016, PRD conducted nearly 200 audits. For example, in the fourth quarter of fiscal year 2016, PRD audited five institutions’ and regional offices’ human resource management functions. During these audits, PRD identified nine deficiencies, one of which pertained to retention incentives. Specifically, it found that one audited institution did not terminate an employee’s retention incentive after the employee had relocated to another institution. To correct the deficiency, the institution cancelled the retention incentive which discontinued future disbursements. According to BOP officials, a bill was generated to recoup the overpayment from the employee. BOP institutions conduct annual operation reviews of internal functions, such as human resources. BOP officials told us that each institution conducts annual operational reviews of various internal functions, such as human resources. According to BOP’s Program Review Guidelines for Human Resource Servicing Offices, during these reviews, institutions are required to review supporting documentation for staff currently receiving an incentive to determine if the incentives are still warranted. If the initial request for the retention incentive was made over the preceding 12 months, institutions are also required to ensure that it was approved. According to BOP officials, the results of these reviews are reported to PRD through the Central Office. DOJ’s Justice Management Division (JMD) audits BOP’s human resources programs. According to BOP officials, JMD conducts audits of component-level human resources programs to determine whether BOP’s systems are compliant with DOJ policy and aligned with DOJ’s Human Capital Strategic Plan. JMD’s most recent audit of BOP’s human resources programs that included a review of BOP’s retention incentives occurred in September 2010 at BOP’s Human Resource Service Center in Grand Prairie, Texas. JMD found that in some cases BOP granted retention incentives prior to the signing of service agreements. JMD also found that BOP lacked documentation to authorize a group retention incentive for employees at its Victorville, California institution. BOP’s written response to the findings stated that JMD incorrectly applied the service agreement requirement, as service agreements were not warranted in the specific case that it identified. Additionally, BOP stated that the documents JMD identified as missing from the case files in question were kept in separate files and not required to be part of the retention incentive application. JMD agreed with BOP’s responses and in January 2013, JMD closed out the audit’s findings noting that these responses satisfied all required corrective actions. While BOP takes a number of steps to determine current workforce needs and how to fill those needs, BOP does not strategically plan for how retention incentives can be used to meet long-term human capital goals. BOP officials stated that planning for human capital needs is conducted at institutions during quarterly workforce utilization meetings or manpower salary meetings. During these meetings, executive staff at the institution discuss the current state of the institution’s workforce. According to the BOP officials, while considering attrition, hiring, and turnover rates, the executive staff decide strategies they will employ to attract and retain employees for their current needs. While officials we spoke with at four institutions have discussed retention incentives at their workforce utilization meetings, details about the content of these discussions ranged. According to these officials and our review of meeting minutes from the four institutions, discussions about retention incentives respond to each institution’s short-term staffing situation rather than address future staffing needs based on an overall strategic human capital plan. For example: USP Atwater officials told us that they review the current turnover rate, budget, projected vacancies, and use of retention incentives at annual budget development meetings. Meeting minutes reflected the following on retention incentives: “retention … still necessary to retain staff and hard-to-fill positions.” FCC Butner is a medical facility that offers retention incentives to all medical officers (all types of doctors) and nurses (practitioners, registered, etc.) at the institution. According to Butner officials, during workforce utilization meetings, Butner officials discuss recruitment and staffing trends for the institution and plans for how to address any staffing challenges. Meeting minutes we reviewed did not indicate specific discussions about the use of retention incentives. FCC Pollock executive staff discuss current institutional salary expenditures and projections and the status of vacant positions at workforce utilization meetings. While meeting minutes we reviewed indicated discussions about projected expenditures for incentive awards, the minutes did not differentiate between retention incentive awards, and other incentive awards such as recruitment or relocation incentive awards. FCI Phoenix officials stated that in their workforce utilization meetings, executive staff discuss salary projections and vacancy statuses. Meeting minutes we reviewed did not indicate specific discussions about the use of retention incentives. BOP decisions about retention incentives are currently not tied to any strategic human capital plan for how to use human capital flexibilities— such as retention incentives—to address their ongoing challenge of retaining staff in hard-to-fill positions. According to officials, retention incentives are awarded on an as-needed basis, determined by an institution’s warden, if funds are available. According to key principles for effective strategic human capital planning, such planning is an important component of an agency’s effort to develop long-term strategies for acquiring, developing, and retaining staff needed for an agency to achieve its goals. Specifically, senior leaders should be involved in developing, communicating, and implementing strategic human capital plans. Within an agency’s strategic human capital plan, the human capital policies, practices, and programs—for example, an agency’s retention incentive program—should clearly link to the human capital and program goals of the organization. By not having a strategic human capital plan that clearly establishes strategies that will be used to achieve specific human capital goals, BOP cannot ensure that its institutions are strategically managing their workforces in a manner that meets the agency’s human capital needs. In August 2017, BOP officials told us that they began drafting a strategic human capital operating plan that will include strategic objectives, action plans, performance objectives and measures, and evaluation/reporting requirements. Officials stated that the plan will also include planning regarding the use of human capital flexibilities, such as retention incentives. BOP officials told us that they anticipate that the strategic human capital operating plan will be a supplement to their workforce utilization meetings and that an agency-wide plan will provide a set of strategies for all institutions to consider. However, BOP could not provide documentation of the project beginning or whether it would include a strategic approach specific to retention incentives. Including retention incentives in BOP’s strategic human capital operating plan would create a roadmap for the agency and the institutions to use to move from being reactive in their current workforce needs—for example, awarding retention incentives on an ad hoc basis when funds are available—to being strategic in how retention incentives are used and to ensure that these and other flexibilities help the agency achieve its long-term workforce goals. From fiscal year 2012 through fiscal year 2016, BOP spent more than $59 million on retention incentives but has not established any measures to evaluate their effectiveness. According to officials, BOP has not evaluated the effectiveness of its use of retention incentives because BOP officials consider a retention incentive successful if an employee does not leave the agency. However, BOP also uses other human capital flexibilities along with retention incentives to help retain staff. For example, BOP uses physician and dental comparability allowances—additional pay to a physician or dentist who enters into an agreement for a specified period of service—to help retain these medical personnel. According to officials, it would otherwise be difficult to compete with private sector salaries without the use of all available incentives. However, BOP has not studied whether or how retention incentives have contributed to employees’ retention in relation to other incentives such as physician and dental comparability allowances. According to our work on strategic human capital management and OPM’s guidance, it is crucial for organizations to evaluate the success of their human capital strategies, such as the use of retention incentives. In measuring the performance of these strategies and their contribution to key programmatic results, agencies can make adjustments, if necessary. For example, agencies can use evaluation results to make targeted investments in certain human capital strategies—such as the use of retention incentives—creating a cycle of strategic workforce management, where evaluation informs planning, planning dictates strategies, and strategies are evaluated for effectiveness. While BOP uses retention incentives to address critical skills gaps—such as with medical professionals—evaluating the effectiveness of retention incentives would help BOP determine whether and how retention incentives, as well as other human capital flexibilities, contribute to an employee’s continued employment at BOP or if adjustments to BOP retention strategies must be made for improved results. BOP officials agreed that evaluating the effectiveness of retention incentives would help them be more strategic about their human capital needs and spending on incentives. By including and implementing such an evaluation in its upcoming strategic human capital operating plan, BOP could better determine if it is making maximum use of its funds to retain the necessary qualified personnel or if changes must be made to most effectively retain its staff. As the largest employer within DOJ with some staff working in remote locations and undesirable conditions, BOP relies on a number of available flexibilities, including retention incentives, to help retain its employees. However, BOP currently lacks a strategic approach for using and evaluating retention incentives to address human capital goals. Given BOP’s ongoing staffing challenges, for example, retaining staff in hard-to- fill medical positions, developing a plan that includes a thoughtful blueprint for using retention incentives could help BOP better anticipate and address staffing needs. Moreover, evaluating its use of retention incentives could help BOP determine whether these incentives are effective or whether adjustments are needed to better retain its employees. By using evaluation results to inform planning, and planning to inform how retention incentives are used, BOP would be better positioned to achieve its long-term human capital goals and address its critical staffing needs. We are making two recommendations to BOP: 1. The Director of BOP should include in the forthcoming strategic human capital operating plan, 1) human capital goals and 2) strategies on how human capital flexibilities—including retention incentives—will be used to meet these goals. (Recommendation 1) 2. The Director of BOP should evaluate the effectiveness of BOP’s use of retention incentives to help determine whether the incentives have helped BOP achieve its human capital goals or if adjustments in retention incentives are needed. (Recommendation 2) We requested comments on a draft of this report from DOJ. In an email received November 15, 2017, the DOJ liaison stated that DOJ concurred with our recommendations. The Department did not provide official written comments to include in our report, but did provide written technical comments, which we incorporated as appropriate. As agreed with your office, unless you publicly announce the contents of this report earlier, we plan no further distribution until 30 days from the report date. At that time, we will send copies of this report to the Attorney General and the Director of BOP. In addition, the report is available at no charge on the GAO website at http://www.gao.gov. If you or your staff have any questions about this report, please contact me at (202) 512-9627 or maurerd@gao.gov. Contact points for our Offices of Congressional Relations and Public Affairs may be found on the last page of this report. GAO staff who made key contributions to this report are listed in Appendix III. This report examines (1) how BOP has used its authority to pay retention incentives; (2) what internal controls are in place for the use of retention incentives; and (3) the extent to which BOP plans for and evaluates the use of retention incentives. To determine how BOP has used its authority to pay retention incentives, we reviewed BOP’s July 2012 report on its use of recruitment, relocation, and retention (3R) incentives. We then obtained underlying retention incentive expenditure data from DOJ’s Justice Management Division because it serves as the focal point for performance and financial information for all Department of Justice components and employees, including BOP. In particular, we obtained employee-level retention incentive payroll data for fiscal years 2012 through 2016. We selected this time period because it includes the most recent five complete fiscal years for which data were available and because we believe five years is sufficient time to identify trends in BOP’s retention incentive expenditures. We analyzed and aggregated the employee-level data by institution, occupation, and employee grade level. To identify trends, we compared per fiscal year expenditures across the various categories of occupations and locations across the five years. Additionally, we categorized institutions by BOP region, institutions that use group retention incentives, and institutions that use individual retention incentives. We also categorized occupations as medical professionals, correctional officers, and all other occupations and compared aggregate retention incentive expenditures for the different groups. Using information from BOP’s website and testimonial evidence from BOP officials on its health care system, for the purposes of this report, we defined medical professionals as BOP employees in occupations that provide medical, dental, and mental health care services and who do not solely provide these services in an administrative function. For the purposes of our analyses, medical professionals are dentists, dental assistants and hygienists, diagnostic radiological technologists, health aid and technicians, medical doctors (including psychiatrists), medical technologists, nurses, pharmacists, pharmacy technicians, physician assistants, and practical nurses and psychologists. To assess the employee-level retention incentive payroll data’s reliability, we obtained and analyzed documentation on systems’ capabilities and data control, interviewed data users and managers responsible for maintaining data, conducted checks for completeness and logical consistency, and compared the employee-level data to aggregated institution-level retention incentive expenditure data from BOP’s Financial Management Information System. We found the employee-level data to be sufficiently reliable for the purpose of this report. Additionally for this objective, we reviewed documents such as the DOJ’s Financial Management Information System Sub-Object Classification Code Guide and the Office of Personnel Management (OPM) Handbook of Occupational Groups and Families to respectively identify the system codes used to track retention incentives expenditures and to identify the names for each occupational series code in the datasets. We also interviewed BOP Human Resource Management headquarters officials to obtain information on the primary purposes for BOP’s use of retention incentives and their views on identified retention incentive expenditures trends. We also interviewed U.S. Department of Health and Human Services’ (HHS) Public Health Service (PHS) officials to better understand how BOP and PHS manage costs, including retention incentive expenditures, for PHS staff assigned to BOP. BOP partners with PHS to acquire medical staff to provide medical care for BOP’s inmate population. BOP reimburses PHS for the costs of compensation and benefits—including retention incentive payments, if applicable—for PHS staff assigned to BOP. PHS has final approval authority for retention incentives paid to PHS staff assigned to BOP facilities. Furthermore, we obtained aggregated retention incentive expenditure data from PHS on the total amount of funds BOP reimbursed PHS for fiscal years 2012 through 2016. For the reliability of PHS’s data, we reviewed the system’s data fields to check that the appropriate fields were used to provide data and interviewed data users and managers to discuss how expenditures are recorded and maintained. We found the PHS data to be sufficiently reliable for the purpose of this report. To identify and describe the internal controls that BOP has in place related to retention incentives, we obtained and analyzed documentation regarding BOP requirements and guidance for the use of retention incentives. We also interviewed officials from BOP’s Central Office who are responsible for the administration, management, and oversight of BOP’s human capital management systems, including retention incentives. We focused on the management and administrative controls used by BOP to review, approve, re-certify, and monitor retention incentives. Additionally, we interviewed the warden and human capital officers at 4 of the 122 institutions to obtain illustrative examples regarding the internal controls in place at these institutions to ensure the proper disbursement of retention incentives. We interviewed BOP officials at Federal Correctional Complex Pollock in Pollock, LA; Federal Correctional Complex Butner in Butner, NC; United States Penitentiary, Atwater in Atwater, CA and Federal Correctional Institution Phoenix, in Phoenix, AZ. These institutions were selected to ensure variation in the number and types of employees receiving retention incentives, BOP region, and security-level. Although the information we obtained from the interviews with officials at these four institutions cannot be generalized to other BOP institutions, these interviews provided important insights and perspectives about the use of retention incentives at BOP institutions. We also reviewed a non-generalizable random sample of 40 retention incentive application packet case files to determine the extent to which these files contained documentation on the internal control activities in place to monitor the application, approval, and funds disbursement processes of BOP’s retention incentive program. To identify our sample, we used employee-level expenditure data to randomly select 40 application files from the universe of BOP employees who received retention incentives from fiscal years 2014 through 2016. Each application file was reviewed by two GAO analysts who each assessed the extent to which each application contained the appropriate justification, approval signatures, and other documentation such as an application checklist and whether the application was an initial or continuation application. To determine the extent to which BOP plans for and evaluates the use of retention incentives, we interviewed BOP officials regarding their experiences with retention incentives, how they use retention incentives to strategically manage their workforce needs, how the agency evaluates the effectiveness of retention incentives, and how retention incentives contribute to BOP’s broader human capital goals. We then compared these efforts to our work on strategic human capital planning, specifically in terms of planning for and evaluating the use of human capital flexibilities. Additionally, we interviewed the warden and human capital officers at four BOP institutions mentioned above to obtain illustrative examples of how workforce planning occurs at these institutions. We also reviewed the DOJ’s Office of Inspector General Report 16-02 “Review of the Federal Bureau of Prisons’ Medical Staffing Challenges” (March 2016) and our past work to better understand the challenges that BOP faces in retaining medical professionals and other staff. Table 2 provides the Bureau of Prisons’ (BOP) fiscal year 2016 retention incentive expenditures by various occupations and groups of occupations, such as medical professionals, correctional officers, and other occupations. A range of occupations are reflected in the table primarily as a result of four California institutions—United States Penitentiary (USP) Atwater, Federal Correctional Institution (FCI) Herlong, FCI Mendota, and Federal Correctional Complex Victorville—providing retention incentives to all employees at General Schedule grades level 12 and below and those in the Federal Wage System. In addition to the contact named above, Dawn Locke (Assistant Director) and Meghan Squires (Analyst-in-Charge) managed the work. Also, David Alexander, Renee Caputo, Willie Commons III, Jamarla Edwards, Robert Goldenkoff, Chelsa Gurkin, Eric Hauswirth, Janice Latimer, Lerone Reid, Rachel Stoiko, and Adam Vogt made significant contributions to this report. + +Now, write a one-page summary of the report. + +Summary: \ No newline at end of file diff --git a/python/llm/dev/benchmark/all-in-one/prompt/continuation/longbench_4k.txt b/python/llm/dev/benchmark/all-in-one/prompt/continuation/longbench_4k.txt new file mode 100644 index 00000000000..64e1923bc3f --- /dev/null +++ b/python/llm/dev/benchmark/all-in-one/prompt/continuation/longbench_4k.txt @@ -0,0 +1,760 @@ +Please complete the code given below. +app/src/main/java/com/matejdro/pebbledialer/callactions/EndCallAction.java +public class EndCallAction extends CallAction +{ + public static final int END_CALL_ACTION_ID = 1; + + private PendingIntent notificationEndCallIntent; + private static Method getITelephonyMethod; + + public EndCallAction(CallModule callModule) + { + super(callModule); + + try { + getITelephonyMethod = TelephonyManager.class.getDeclaredMethod("getITelephony", (Class[]) null); + getITelephonyMethod.setAccessible(true); + } catch (NoSuchMethodException e) { + Timber.e(e, "iTelephony end not supported on your phone!"); + } catch (Exception e) { + Timber.e(e, "Error while acquiring iTelephony"); + Crashlytics.logException(e); + } + + } + + public void registerNotificationEndCallIntent(PendingIntent notificationAnswerIntent) + { + this.notificationEndCallIntent = notificationAnswerIntent; + } + + @Override + public void executeAction() + { + getCallModule().setCloseAutomaticallyAfterThisCall(true); + + if (getCallModule().getService().getGlobalSettings().getBoolean("rootMode", false)) + { + Timber.d("Ending call using root method..."); + try { + Runtime.getRuntime().exec(new String[] {"su", "-c", "input keyevent 6"}); + return; + } catch (IOException e) { + e.printStackTrace(); + } + } + + if (getCallModule().getCallState() == CallModule.CallState.RINGING && notificationEndCallIntent != null) + { + Timber.d("Ending call using notification method..."); + + try { + notificationEndCallIntent.send(); + return; + } catch (PendingIntent.CanceledException e) { + } + } + + if (getITelephonyMethod != null) + { + Timber.d("Ending call using generic iTelephony method..."); + try + { + ITelephony iTelephony = (ITelephony) getITelephonyMethod.invoke(getCallModule().getService().getSystemService(Context.TELEPHONY_SERVICE), (Object[]) null); + iTelephony.endCall(); + return; + } + catch (SecurityException e) + { + Timber.e("Cannot decline call, no CALL_PHONE permission."); + } + catch (Exception e) { + Timber.e(e, "Error while invoking iTelephony.endCall()"); + Crashlytics.logException(e); + } + } + + Timber.e("All end call options failed! Nothing is supported."); + } + + @Override + public void onCallEnd() + { + notificationEndCallIntent = null; //Reset intent (there will be new intent at next call) + } + + @Override + public int getIcon() + { + return CallAction.ICON_BUTTON_END_CALL; + } + + public static EndCallAction get(CallModule callModule) + { + return (EndCallAction) callModule.getCallAction(END_CALL_ACTION_ID); + } +} +app/src/main/java/com/matejdro/pebbledialer/callactions/AnswerCallAction.java +public class AnswerCallAction extends CallAction +{ + public static final int ANSWER_ACTION_ID = 0; + + private PendingIntent notificationAnswerIntent; + + public AnswerCallAction(CallModule callModule) + { + super(callModule); + } + + public void registerNotificationAnswerIntent(PendingIntent notificationAnswerIntent) + { + this.notificationAnswerIntent = notificationAnswerIntent; + } + + @Override + public void executeAction() + { + if (getCallModule().getCallState() != CallModule.CallState.RINGING) + return; + + if (getCallModule().getService().getGlobalSettings().getBoolean("rootMode", false)) + { + Timber.d("Answering using root method..."); + try { + Runtime.getRuntime().exec(new String[] {"su", "-c", "input keyevent 5"}); + return; + } catch (IOException e) { + e.printStackTrace(); + } + + } + + if (notificationAnswerIntent != null) + { + Timber.d("Answering using notification method..."); + + try { + notificationAnswerIntent.send(); + return; + } catch (PendingIntent.CanceledException e) { + } + } + + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) + { + answerNativelyOreo(); + } + else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) + { + answerUsingMediaServer(); + } + else + { + Timber.d("Answering using generic headset hook method..."); + Intent buttonUp = new Intent(Intent.ACTION_MEDIA_BUTTON); + buttonUp.putExtra(Intent.EXTRA_KEY_EVENT, new KeyEvent(KeyEvent.ACTION_UP, KeyEvent.KEYCODE_HEADSETHOOK)); + getCallModule().getService().sendOrderedBroadcast(buttonUp, "android.permission.CALL_PRIVILEGED"); + } + + } + + @TargetApi(Build.VERSION_CODES.O) + private void answerNativelyOreo() { + TelecomManager telecomManager + = (TelecomManager) getCallModule().getService().getSystemService(Context.TELECOM_SERVICE); + + Timber.d("Answering natively with Oreo."); + + try { + telecomManager.acceptRingingCall(); + } catch (SecurityException e) { + Timber.e("No accept call permission!"); + } + } + + @TargetApi(Build.VERSION_CODES.LOLLIPOP) + private void answerUsingMediaServer() + { + Timber.d("Answering using media server method..."); + + MediaSessionManager mediaSessionManager = (MediaSessionManager) getCallModule().getService().getSystemService(Context.MEDIA_SESSION_SERVICE); + + try { + List mediaControllerList = mediaSessionManager.getActiveSessions + (new ComponentName(getCallModule().getService(), JellybeanNotificationListener.class)); + + for (MediaController m : mediaControllerList) { + if ("com.android.server.telecom".equals(m.getPackageName())) { + Timber.d("Found telephony media controller!"); + m.dispatchMediaButtonEvent(new KeyEvent(KeyEvent.ACTION_UP, KeyEvent.KEYCODE_HEADSETHOOK)); + break; + } + } + } catch (SecurityException e) { + Timber.e("Notification service not running!"); + } + } + + @Override + public void onCallEnd() + { + notificationAnswerIntent = null; //Reset intent (there will be new intent at next call) + } + + @Override + public int getIcon() + { + return CallAction.ICON_BUTTON_ANSWER; + } + + public static AnswerCallAction get(CallModule callModule) + { + return (AnswerCallAction) callModule.getCallAction(ANSWER_ACTION_ID); + } +} +app/src/main/java/com/matejdro/pebbledialer/callactions/SMSReplyAction.java +public class SMSReplyAction extends CallAction +{ + public static final int SMS_REPLY_ACTION_ID = 6; + + public SMSReplyAction(CallModule callModule) + { + super(callModule); + } + + @Override + public void executeAction() + { + ToggleRingerAction toggleRingerAction = ToggleRingerAction.get(getCallModule()); + toggleRingerAction.mute(); + + SMSReplyModule smsReplyModule = SMSReplyModule.get(getCallModule().getService()); + smsReplyModule.startSMSProcess(getCallModule().getNumber()); + + getCallModule().setCloseAutomaticallyAfterThisCall(false); + } + + @Override + public void onCallEnd() + { + } + + @Override + public int getIcon() + { + return CallAction.ICON_BUTTON_END_CALL; + } + + public static SMSReplyAction get(CallModule callModule) + { + return (SMSReplyAction) callModule.getCallAction(SMS_REPLY_ACTION_ID); + } +} +app/src/main/java/com/matejdro/pebbledialer/callactions/ToggleMicrophoneAction.java +public class ToggleMicrophoneAction extends CallAction +{ + public static final int TOGGLE_MICROPHONE_ACTION_ID = 3; + + private boolean microphoneMuted = false; + + public ToggleMicrophoneAction(CallModule callModule) + { + super(callModule); + } + + @Override + public void executeAction() + { + if (getCallModule().getCallState() != CallModule.CallState.ESTABLISHED) + return; + + microphoneMuted = !microphoneMuted; + + if (getCallModule().getService().getGlobalSettings().getBoolean("rootMode", false)) + { + String muteCommand; + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) + muteCommand = "input keyevent 79"; + else + muteCommand = "input keyevent 91"; + try { + Runtime.getRuntime().exec(new String[] {"su", "-c", muteCommand}); + } catch (IOException e) { + e.printStackTrace(); + } + } + else + { + AudioManager audioManager = (AudioManager) getCallModule().getService().getSystemService(Context.AUDIO_SERVICE); + audioManager.setMicrophoneMute(microphoneMuted); + } + + getCallModule().updatePebble(); + } + + @Override + public int getIcon() + { + return microphoneMuted ? CallAction.ICON_BUTTON_MIC_OFF : CallAction.ICON_BUTTON_MIC_ON; + } + + public static ToggleMicrophoneAction get(CallModule callModule) + { + return (ToggleMicrophoneAction) callModule.getCallAction(TOGGLE_MICROPHONE_ACTION_ID); + } +} +app/src/main/java/com/matejdro/pebbledialer/callactions/ToggleRingerAction.java +public class ToggleRingerAction extends CallAction +{ + public static final int TOGGLE_RINGER_ACTION_ID = 2; + + private boolean isMutedViaAudioManager = false; + private int prevRingerMode = AudioManager.RINGER_MODE_NORMAL; + + public ToggleRingerAction(CallModule callModule) + { + super(callModule); + } + + @Override + public void executeAction() + { + if (getCallModule().getCallState() != CallModule.CallState.RINGING) + return; + + AudioManager audioManager = (AudioManager) getCallModule().getService().getSystemService(Context.AUDIO_SERVICE); + + getCallModule().setVibration(false); + + if (!isMutedViaAudioManager) + { + if (getCallModule().getService().getGlobalSettings().getBoolean("rootMode", false)) + { + Timber.d("Muting using root method..."); + try { + Runtime.getRuntime().exec(new String[] {"su", "-c", "input keyevent " + KeyEvent.KEYCODE_VOLUME_DOWN}); + } catch (IOException e) { + e.printStackTrace(); + } + + } + else if (canMuteRinger(getCallModule().getService())) + { + isMutedViaAudioManager = true; + prevRingerMode = audioManager.getRingerMode(); + + audioManager.setStreamSolo(AudioManager.STREAM_MUSIC, true); + audioManager.setRingerMode(AudioManager.RINGER_MODE_SILENT); + } + } + else if (canMuteRinger(getCallModule().getService())) + { + isMutedViaAudioManager = false; + audioManager.setStreamSolo(AudioManager.STREAM_MUSIC, false); + audioManager.setRingerMode(prevRingerMode); + } + + getCallModule().updatePebble(); + } + + public void mute() + { + if (!isMutedViaAudioManager) + executeAction(); + } + + public static boolean canMuteRinger(Context context) + { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.M) + return true; + + NotificationManager notificationManager = (NotificationManager) context.getSystemService(Context.NOTIFICATION_SERVICE); + return notificationManager.isNotificationPolicyAccessGranted(); + } + + @Override + public void onCallEnd() + { if (isMutedViaAudioManager && canMuteRinger(getCallModule().getService())) + { + AudioManager audioManager = (AudioManager) getCallModule().getService().getSystemService(Context.AUDIO_SERVICE); + isMutedViaAudioManager = false; + audioManager.setStreamSolo(AudioManager.STREAM_MUSIC, false); + audioManager.setRingerMode(prevRingerMode); + } + + getCallModule().setVibration(true); + } + + @Override + public int getIcon() + { + return isMutedViaAudioManager ? CallAction.ICON_BUTTON_SPEAKER_OFF : CallAction.ICON_BUTTON_SPEKAER_ON; + } + + public static ToggleRingerAction get(CallModule callModule) + { + return (ToggleRingerAction) callModule.getCallAction(TOGGLE_RINGER_ACTION_ID); + } +} +app/src/main/java/com/matejdro/pebbledialer/callactions/AnswerCallWithSpeakerAction.java +public class AnswerCallWithSpeakerAction extends CallAction +{ + public static final int ANSWER_WITH_SPEAKER_ACTION_ID = 5; + + private boolean enableSpeakerImmediately = false; + + public AnswerCallWithSpeakerAction(CallModule callModule) + { + super(callModule); + } + + + @Override + public void executeAction() + { + if (getCallModule().getCallState() != CallModule.CallState.RINGING) + return; + + enableSpeakerImmediately = true; + AnswerCallAction.get(getCallModule()).executeAction(); + } + + @Override + public void onCallEnd() + { + enableSpeakerImmediately = false; //Reset intent (there will be new intent at next call) + } + + @Override + public void onPhoneOffhook() + { + if (enableSpeakerImmediately) + { + ToggleSpeakerAction speakerAction = ToggleSpeakerAction.get(getCallModule()); + + if (!speakerAction.isSpeakerphoneEnabled()) + speakerAction.executeAction(); + } + } + + @Override + public int getIcon() + { + return CallAction.ICON_BUTTON_ANSWER; + } + + public static AnswerCallWithSpeakerAction get(CallModule callModule) + { + return (AnswerCallWithSpeakerAction) callModule.getCallAction(ANSWER_WITH_SPEAKER_ACTION_ID); + } +} +app/src/main/java/com/matejdro/pebbledialer/callactions/VolumeDownAction.java +public class VolumeDownAction extends CallAction +{ + public static final int VOLUME_DOWN_ACTION_ID = 7; + + public VolumeDownAction(CallModule callModule) + { + super(callModule); + } + + @Override + public void executeAction() + { + if (getCallModule().getCallState() != CallModule.CallState.ESTABLISHED) + return; + + AudioManager audioManager = (AudioManager) getCallModule().getService().getSystemService(Context.AUDIO_SERVICE); + audioManager.adjustStreamVolume(AudioManager.STREAM_VOICE_CALL, AudioManager.ADJUST_LOWER, 0); + } + + + + @Override + public int getIcon() + { + return CallAction.ICON_BUTTON_VOLUME_DOWN; + } +} +app/src/main/java/com/matejdro/pebbledialer/callactions/CallAction.java +public abstract class CallAction +{ + public static final int ICON_BUTTON_ANSWER = 0; + public static final int ICON_BUTTON_END_CALL = 1; + public static final int ICON_BUTTON_MIC_ON = 2; + public static final int ICON_BUTTON_MIC_OFF = 3; + public static final int ICON_BUTTON_SPEKAER_ON = 4; + public static final int ICON_BUTTON_SPEAKER_OFF = 5; + public static final int ICON_BUTTON_VOLUME_DOWN = 6; + public static final int ICON_BUTTON_VOLUME_UP = 7; + public static final int ICON_BLANK = 0xFF; + + private CallModule callModule; + + public CallAction(CallModule callModule) + { + this.callModule = callModule; + } + + public CallModule getCallModule() + { + return callModule; + } + + public void onPhoneOffhook() + { + + } + + public void onCallRinging() + { + + } + + public void onCallEnd() + { + + } + + public abstract void executeAction(); + public abstract int getIcon(); +} +app/src/main/java/com/matejdro/pebbledialer/notifications/JellybeanNotificationListener.java +@TargetApi(value = Build.VERSION_CODES.JELLY_BEAN_MR2) +public class JellybeanNotificationListener extends NotificationListenerService { + private static JellybeanNotificationListener instance = null; + + @Override + public void onDestroy() { + Timber.d("Notification Listener stopped..."); + super.onDestroy(); + + instance = null; + } + + @Override + public void onCreate() { + Timber.d("Creating Notification Listener..."); + super.onCreate(); + + instance = this; + } + + public static boolean isActive() + { + return instance != null; + } + + @TargetApi(value = Build.VERSION_CODES.LOLLIPOP) + public static boolean isPhoneInDoNotInterrupt() + { + if (instance == null) + return false; + + int interruptionFilter = instance.getCurrentInterruptionFilter(); + Timber.d("Interrupt filter: %d", interruptionFilter); + return interruptionFilter != NotificationListenerService.INTERRUPTION_FILTER_ALL && interruptionFilter != 0; + } + + @Override + public void onNotificationPosted(final StatusBarNotification sbn) { + Timber.d("Got new jellybean notification"); + NotificationHandler.newNotification(JellybeanNotificationListener.this, sbn.getPackageName(), sbn.getNotification()); + + + } + + @Override + public void onNotificationRemoved(StatusBarNotification sbn) { + } +} +app/src/main/java/com/matejdro/pebbledialer/callactions/ToggleSpeakerAction.java +public class ToggleSpeakerAction extends CallAction +{ + public static final int TOGGLE_SPEAKER_ACTION_ID = 4; + + private boolean speakerphoneEnabled = false; + + public ToggleSpeakerAction(CallModule callModule) + { + super(callModule); + } + + @Override + public void executeAction() + { + if (getCallModule().getCallState() != CallModule.CallState.ESTABLISHED) + return; + + AudioManager audioManager = (AudioManager) getCallModule().getService().getSystemService(Context.AUDIO_SERVICE); + + speakerphoneEnabled = !speakerphoneEnabled; + audioManager.setSpeakerphoneOn(speakerphoneEnabled); + + getCallModule().updatePebble(); + } + + public boolean isSpeakerphoneEnabled() + { + return speakerphoneEnabled; + } + + private void updateSpeakerphoneEnabled() + { + AudioManager audioManager = (AudioManager) getCallModule().getService().getSystemService(Context.AUDIO_SERVICE); + speakerphoneEnabled = audioManager.isSpeakerphoneOn(); + } + + @Override + public void onPhoneOffhook() + { + updateSpeakerphoneEnabled(); + } + + @Override + public int getIcon() + { + return speakerphoneEnabled ? ICON_BUTTON_SPEKAER_ON : ICON_BUTTON_SPEAKER_OFF; + } + + public static ToggleSpeakerAction get(CallModule callModule) + { + return (ToggleSpeakerAction) callModule.getCallAction(TOGGLE_SPEAKER_ACTION_ID); + } +} +app/src/main/java/com/matejdro/pebbledialer/callactions/VolumeUpAction.java +public class VolumeUpAction extends CallAction +{ + public static final int VOLUME_UP_ACTION_ID = 8; + + public VolumeUpAction(CallModule callModule) + { + super(callModule); + } + + @Override + public void executeAction() + { + if (getCallModule().getCallState() != CallModule.CallState.ESTABLISHED) + return; + + AudioManager audioManager = (AudioManager) getCallModule().getService().getSystemService(Context.AUDIO_SERVICE); + audioManager.adjustStreamVolume(AudioManager.STREAM_VOICE_CALL, AudioManager.ADJUST_RAISE, 0); + } + + + + @Override + public int getIcon() + { + return CallAction.ICON_BUTTON_VOLUME_UP; + } +} +app/src/main/java/com/matejdro/pebbledialer/callactions/DummyAction.java +public class DummyAction extends CallAction +{ + public static final int DUMMY_ACTION_ID = 999; + + + public DummyAction(CallModule callModule) + { + super(callModule); + + } + + @Override + public void executeAction() + { + } + + @Override + public int getIcon() + { + return CallAction.ICON_BLANK; + } + + public static DummyAction get(CallModule callModule) + { + return (DummyAction) callModule.getCallAction(DUMMY_ACTION_ID); + } +} +package com.matejdro.pebbledialer.modules; +import android.app.PendingIntent; +import android.content.Intent; +import android.content.SharedPreferences; +import android.database.Cursor; +import android.graphics.Bitmap; +import android.net.Uri; +import android.os.Build; +import android.provider.ContactsContract; +import android.provider.MediaStore; +import android.service.notification.NotificationListenerService; +import android.telephony.TelephonyManager; +import android.util.SparseArray; +import com.getpebble.android.kit.util.PebbleDictionary; +import com.matejdro.pebblecommons.pebble.CommModule; +import com.matejdro.pebblecommons.pebble.PebbleCommunication; +import com.matejdro.pebblecommons.pebble.PebbleImageToolkit; +import com.matejdro.pebblecommons.pebble.PebbleTalkerService; +import com.matejdro.pebblecommons.pebble.PebbleUtil; +import com.matejdro.pebblecommons.util.ContactUtils; +import com.matejdro.pebblecommons.util.Size; +import com.matejdro.pebblecommons.util.TextUtil; +import com.matejdro.pebblecommons.vibration.PebbleVibrationPattern; +import com.matejdro.pebbledialer.callactions.AnswerCallAction; +import com.matejdro.pebbledialer.callactions.AnswerCallWithSpeakerAction; +import com.matejdro.pebbledialer.callactions.CallAction; +import com.matejdro.pebbledialer.callactions.DummyAction; +import com.matejdro.pebbledialer.callactions.EndCallAction; +import com.matejdro.pebbledialer.callactions.SMSReplyAction; +import com.matejdro.pebbledialer.callactions.ToggleMicrophoneAction; +import com.matejdro.pebbledialer.callactions.ToggleRingerAction; +import com.matejdro.pebbledialer.callactions.ToggleSpeakerAction; +import com.matejdro.pebbledialer.callactions.VolumeDownAction; +import com.matejdro.pebbledialer.callactions.VolumeUpAction; +import com.matejdro.pebbledialer.notifications.JellybeanNotificationListener; +import java.io.IOException; +import java.util.Calendar; +import java.util.List; +import timber.log.Timber; + + + + + +public class CallModule extends CommModule +{ + public static final String INTENT_CALL_STATUS = "CallStatus"; + public static final String INTENT_ACTION_FROM_NOTIFICATION = "ActionFromNotification"; + + public static int MODULE_CALL = 1; + + private SparseArray actions = new SparseArray(); + + private boolean updateRequired; + private boolean identityUpdateRequired; + private boolean callerNameUpdateRequired; + private int callerImageNextByte = -1; + + private String number = "Outgoing Call"; + private String name = null; + private String type = null; + private Bitmap callerImage = null; + private byte[] callerImageBytes; + + private CallState callState = CallState.NO_CALL; + + private boolean vibrating; + private boolean closeAutomaticallyAfterThisCall = true; + + long callStartTime; + + public CallModule(PebbleTalkerService service) + { + super(service); + + service.registerIntent(INTENT_CALL_STATUS, this); + service.registerIntent(INTENT_ACTION_FROM_NOTIFICATION, this); + + registerCallAction(new AnswerCallAction(this), AnswerCallAction.ANSWER_ACTION_ID); + registerCallAction(new EndCallAction(this), EndCallAction.END_CALL_ACTION_ID);Next line of code: diff --git a/python/llm/dev/benchmark/all-in-one/run.py b/python/llm/dev/benchmark/all-in-one/run.py index 7a6d7d70b70..d94f45d2efc 100644 --- a/python/llm/dev/benchmark/all-in-one/run.py +++ b/python/llm/dev/benchmark/all-in-one/run.py @@ -79,12 +79,13 @@ def run_model_in_thread(model, in_out, tokenizer, result, warm_up, num_beams, in def get_continuation_input_str(in_len, tokenizer): # keep 'utf-8' as character encoding mode - if in_len <= 2048: - input_str = open("prompt/summarize/cnn_39.txt", 'r', encoding='utf-8').read() - question = "Can you please summarize this article?" - prompt_format = "{}\nQuestion: {}\n" - full_prompt = prompt_format.format(input_str, question) - input_ids = tokenizer.encode(full_prompt, return_tensors="pt") + if in_len > 128 and in_len <= 4096: + if in_len <= 2048: + input_str = open("prompt/continuation/longbench_2k.txt", 'r', encoding='utf-8').read() + else: + input_str = open("prompt/continuation/longbench_4k.txt", 'r', encoding='utf-8').read() + + input_ids = tokenizer.encode(input_str, return_tensors="pt") if input_ids.shape[1] < in_len: return open(f"prompt/continuation/8192.txt", 'r', encoding='utf-8').read() else: From 7d81e6f78bfb3d6f1052e297789381821e4f131c Mon Sep 17 00:00:00 2001 From: Yuwen Hu Date: Fri, 16 Aug 2024 14:20:58 +0800 Subject: [PATCH 2/7] Small fix --- python/llm/dev/benchmark/all-in-one/run.py | 29 +++++++++++----------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/python/llm/dev/benchmark/all-in-one/run.py b/python/llm/dev/benchmark/all-in-one/run.py index d94f45d2efc..9e5586dedf4 100644 --- a/python/llm/dev/benchmark/all-in-one/run.py +++ b/python/llm/dev/benchmark/all-in-one/run.py @@ -77,21 +77,22 @@ def run_model_in_thread(model, in_out, tokenizer, result, warm_up, num_beams, in actual_in_len, actual_out_len, load_time, model.peak_memory]) -def get_continuation_input_str(in_len, tokenizer): +def get_continuation_input_str(in_len, tokenizer=None): # keep 'utf-8' as character encoding mode - if in_len > 128 and in_len <= 4096: - if in_len <= 2048: - input_str = open("prompt/continuation/longbench_2k.txt", 'r', encoding='utf-8').read() - else: - input_str = open("prompt/continuation/longbench_4k.txt", 'r', encoding='utf-8').read() + if tokenizer is not None: + if in_len > 128 and in_len <= 4096: + if in_len <= 2048: + input_str = open("prompt/continuation/longbench_2k.txt", 'r', encoding='utf-8').read() + else: + input_str = open("prompt/continuation/longbench_4k.txt", 'r', encoding='utf-8').read() - input_ids = tokenizer.encode(input_str, return_tensors="pt") - if input_ids.shape[1] < in_len: - return open(f"prompt/continuation/8192.txt", 'r', encoding='utf-8').read() - else: - half_idx = in_len // 2 - input_ids_truncated = torch.cat((input_ids[:, :half_idx], input_ids[:, -(in_len-half_idx):]), dim=1) - return tokenizer.batch_decode(input_ids_truncated)[0] + input_ids = tokenizer.encode(input_str, return_tensors="pt") + if input_ids.shape[1] < in_len: + return open(f"prompt/continuation/8192.txt", 'r', encoding='utf-8').read() + else: + half_idx = in_len // 2 + input_ids_truncated = torch.cat((input_ids[:, :half_idx], input_ids[:, -(in_len-half_idx):]), dim=1) + return tokenizer.batch_decode(input_ids_truncated)[0] return open(f"prompt/continuation/8192.txt", 'r', encoding='utf-8').read() @@ -240,7 +241,7 @@ def run_native_int4(repo_id, in_out_len = in_out.split("-") in_len = int(in_out_len[0]) out_len = int(in_out_len[1]) - input_str = get_continuation_input_str(in_len, tokenizer) + input_str = get_continuation_input_str(in_len) # As different tokenizer has different encodings, # slice the input_ids to ensure the prompt length is required length. n_ctx = in_len + out_len if in_len + out_len > 512 else 512 From bca2b28c17a42a09246b0bf1a24bfe84da2410b6 Mon Sep 17 00:00:00 2001 From: Yuwen Hu Date: Fri, 16 Aug 2024 14:52:01 +0800 Subject: [PATCH 3/7] Add pure-text benchmark support for minicpm-v-2_6 --- python/llm/dev/benchmark/all-in-one/run.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/python/llm/dev/benchmark/all-in-one/run.py b/python/llm/dev/benchmark/all-in-one/run.py index 9e5586dedf4..c39a6e2c852 100644 --- a/python/llm/dev/benchmark/all-in-one/run.py +++ b/python/llm/dev/benchmark/all-in-one/run.py @@ -48,6 +48,8 @@ QWENVL_IDS = ['Qwen/Qwen-VL-Chat'] +MINICPM_V_IDS = ['openbmb/MiniCPM-V-2_6'] + results = [] excludes = [] @@ -480,11 +482,13 @@ def run_transformer_int4_gpu(repo_id, trust_remote_code=True, use_cache=True, torch_dtype=torch_dtype).eval() tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True, cpu_embedding=cpu_embedding) + model = model.to('xpu') elif origin_repo_id in LLAMA_IDS: model = AutoModelForCausalLM.from_pretrained(model_path, load_in_low_bit=low_bit, trust_remote_code=True, use_cache=True, cpu_embedding=cpu_embedding, torch_dtype=torch_dtype).eval() tokenizer = LlamaTokenizer.from_pretrained(model_path, trust_remote_code=True) + model = model.to('xpu') elif origin_repo_id in PHI3VISION_IDS: model = AutoModelForCausalLM.from_pretrained(model_path, optimize_model=True, load_in_low_bit=low_bit, _attn_implementation="eager", @@ -492,6 +496,15 @@ def run_transformer_int4_gpu(repo_id, trust_remote_code=True, use_cache=True, cpu_embedding=cpu_embedding, torch_dtype=torch_dtype).eval() tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) + model = model.to('xpu') + elif origin_repo_id in MINICPM_V_IDS: + model = AutoModel.from_pretrained(model_path, load_in_low_bit=low_bit, optimize_model=True, + modules_to_not_convert=["vpm", "resampler"], + trust_remote_code=True, use_cache=True, + torch_dtype=torch_dtype).eval() + tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) + model = model.to('xpu') + model = model.llm else: if "4bit" in repo_id: model = AutoModelForCausalLM.load_low_bit(model_path, optimize_model=True, @@ -512,10 +525,9 @@ def run_transformer_int4_gpu(repo_id, trust_remote_code=True, use_cache=True, cpu_embedding=cpu_embedding, torch_dtype=torch_dtype).eval() + model = model.to('xpu') tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) - model = model.to('xpu') - end = time.perf_counter() load_time = end - st print(">> loading of model costs {}s and {}GB".format(load_time, torch.xpu.memory.memory_reserved()/(1024**3))) From 5684a4589ab79d1ede03d0ebfd64729e7fa8c0f1 Mon Sep 17 00:00:00 2001 From: Yuwen Hu Date: Fri, 16 Aug 2024 16:29:16 +0800 Subject: [PATCH 4/7] Support lookahead for model.llm generate of minicpmv --- python/llm/src/ipex_llm/transformers/model.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/llm/src/ipex_llm/transformers/model.py b/python/llm/src/ipex_llm/transformers/model.py index d9dd6354970..7290acaed88 100644 --- a/python/llm/src/ipex_llm/transformers/model.py +++ b/python/llm/src/ipex_llm/transformers/model.py @@ -396,6 +396,8 @@ def from_pretrained(cls, from .lookup import lookup_generate import types model.lookup_generate = types.MethodType(lookup_generate, model) + if model.config.model_type == "minicpmv" and hasattr(model, 'llm'): + model.llm.lookup_generate = types.MethodType(lookup_generate, model.llm) else: # load default model = cls.HF_Model.from_pretrained(*args, **kwargs) From a44ab70ca01ff81929c9ce1e1a5a032b7df1c101 Mon Sep 17 00:00:00 2001 From: Yuwen Hu Date: Fri, 16 Aug 2024 16:53:24 +0800 Subject: [PATCH 5/7] Add prompt reference --- python/llm/dev/benchmark/all-in-one/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/llm/dev/benchmark/all-in-one/README.md b/python/llm/dev/benchmark/all-in-one/README.md index f1927e8e7d1..d17332b472c 100644 --- a/python/llm/dev/benchmark/all-in-one/README.md +++ b/python/llm/dev/benchmark/all-in-one/README.md @@ -4,6 +4,8 @@ All in one benchmark test allows users to test all the benchmarks and record the Before running, make sure to have [ipex-llm](../../../../../README.md) installed. +> The prompts for benchmarking are from datasets [abisee/cnn_dailymail](https://huggingface.co/datasets/abisee/cnn_dailymail), [Open-Orca/OpenOrca](https://huggingface.co/datasets/Open-Orca/OpenOrca), [THUDM/LongBench](https://huggingface.co/datasets/THUDM/LongBench), etc. + ## Dependencies ```bash From 9d807af6cde009864810213f9e74cb0536fcba58 Mon Sep 17 00:00:00 2001 From: Yuwen Hu Date: Fri, 16 Aug 2024 17:02:40 +0800 Subject: [PATCH 6/7] Small update --- python/llm/dev/benchmark/all-in-one/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/llm/dev/benchmark/all-in-one/run.py b/python/llm/dev/benchmark/all-in-one/run.py index c39a6e2c852..ff57b6eca81 100644 --- a/python/llm/dev/benchmark/all-in-one/run.py +++ b/python/llm/dev/benchmark/all-in-one/run.py @@ -525,8 +525,8 @@ def run_transformer_int4_gpu(repo_id, trust_remote_code=True, use_cache=True, cpu_embedding=cpu_embedding, torch_dtype=torch_dtype).eval() - model = model.to('xpu') tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) + model = model.to('xpu') end = time.perf_counter() load_time = end - st From 3659ee6f7e19ea1e6d296415b2aeb30f9a74ff93 Mon Sep 17 00:00:00 2001 From: Yuwen Hu Date: Fri, 16 Aug 2024 17:16:15 +0800 Subject: [PATCH 7/7] Small fix --- python/llm/dev/benchmark/all-in-one/run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/llm/dev/benchmark/all-in-one/run.py b/python/llm/dev/benchmark/all-in-one/run.py index ff57b6eca81..a4355dd95b7 100644 --- a/python/llm/dev/benchmark/all-in-one/run.py +++ b/python/llm/dev/benchmark/all-in-one/run.py @@ -480,7 +480,7 @@ def run_transformer_int4_gpu(repo_id, else: model = AutoModel.from_pretrained(model_path, load_in_low_bit=low_bit, optimize_model=True, trust_remote_code=True, use_cache=True, - torch_dtype=torch_dtype).eval() + cpu_embedding=cpu_embedding, torch_dtype=torch_dtype).eval() tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True, cpu_embedding=cpu_embedding) model = model.to('xpu') elif origin_repo_id in LLAMA_IDS: @@ -501,7 +501,7 @@ def run_transformer_int4_gpu(repo_id, model = AutoModel.from_pretrained(model_path, load_in_low_bit=low_bit, optimize_model=True, modules_to_not_convert=["vpm", "resampler"], trust_remote_code=True, use_cache=True, - torch_dtype=torch_dtype).eval() + cpu_embedding=cpu_embedding, torch_dtype=torch_dtype).eval() tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) model = model.to('xpu') model = model.llm