<?xml version="1.0" encoding="us-ascii" standalone="yes"?>
<!-- DFIN ActiveDisclosure(SM) XBRL Linkbase Document - http://www.dfinsolutions.com/ -->
<!-- Creation Date      : 2021-03-30T19:24:16.6036447+00:00 -->
<!-- Version            : 5.0.1.571 -->
<!-- Package ID         : 728bde0f97ea4bf0b7384349263600e3 -->
<!-- Copyright (c) 2021 Donnelley Financial Solutions, Inc. All Rights Reserved. -->
<link:linkbase xmlns:link="http://www.xbrl.org/2003/linkbase" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.xbrl.org/2003/linkbase http://www.xbrl.org/2003/xbrl-linkbase-2003-12-31.xsd">
  <link:roleRef roleURI="http://www.xbrl.org/2009/role/negatedLabel" xlink:href="http://www.xbrl.org/lrr/role/negated-2009-12-16.xsd#negatedLabel" xlink:type="simple" />
  <link:roleRef roleURI="http://www.xbrl.org/2009/role/negatedNetLabel" xlink:href="http://www.xbrl.org/lrr/role/negated-2009-12-16.xsd#negatedNetLabel" xlink:type="simple" />
  <link:roleRef roleURI="http://www.xbrl.org/2009/role/negatedPeriodEndLabel" xlink:href="http://www.xbrl.org/lrr/role/negated-2009-12-16.xsd#negatedPeriodEndLabel" xlink:type="simple" />
  <link:roleRef roleURI="http://www.xbrl.org/2009/role/negatedPeriodStartLabel" xlink:href="http://www.xbrl.org/lrr/role/negated-2009-12-16.xsd#negatedPeriodStartLabel" xlink:type="simple" />
  <link:roleRef roleURI="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:href="http://www.xbrl.org/lrr/role/negated-2009-12-16.xsd#negatedTerseLabel" xlink:type="simple" />
  <link:roleRef roleURI="http://www.xbrl.org/2009/role/negatedTotalLabel" xlink:href="http://www.xbrl.org/lrr/role/negated-2009-12-16.xsd#negatedTotalLabel" xlink:type="simple" />
  <link:roleRef roleURI="http://www.xbrl.org/2009/role/netLabel" xlink:href="http://www.xbrl.org/lrr/role/net-2009-12-16.xsd#netLabel" xlink:type="simple" />
  <link:roleRef roleURI="http://fasb.org/us-gaap/role/label/axisDefault" xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-roles-2020-01-31.xsd#axisDefault" xlink:type="simple" />
  <link:labelLink xlink:role="http://www.xbrl.org/2003/role/link" xlink:type="extended">
    <link:loc xlink:href="https://xbrl.sec.gov/dei/2020/dei-2020-01-31.xsd#dei_EntityCentralIndexKey" xlink:type="locator" xlink:label="dei_EntityCentralIndexKey" />
    <link:label xml:lang="en-US" xlink:label="dei_EntityCentralIndexKey_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Entity Central Index Key</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="dei_EntityCentralIndexKey" xlink:to="dei_EntityCentralIndexKey_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_WarrantDividendsIncomeStatementImpact" xlink:type="locator" xlink:label="pbhc_WarrantDividendsIncomeStatementImpact" />
    <link:label xml:lang="en-US" xlink:label="pbhc_WarrantDividendsIncomeStatementImpact_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Warrant dividends, income statement impact.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_WarrantDividendsIncomeStatementImpact" xlink:to="pbhc_WarrantDividendsIncomeStatementImpact_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_EquitySecuritiesAvailableForSaleAccumulatedGrossUnrealizedGainBeforeTax" xlink:type="locator" xlink:label="pbhc_EquitySecuritiesAvailableForSaleAccumulatedGrossUnrealizedGainBeforeTax" />
    <link:label xml:lang="en-US" xlink:label="pbhc_EquitySecuritiesAvailableForSaleAccumulatedGrossUnrealizedGainBeforeTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Equity securities, available-for-sale, accumulated gross unrealized gain, before tax.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_EquitySecuritiesAvailableForSaleAccumulatedGrossUnrealizedGainBeforeTax" xlink:to="pbhc_EquitySecuritiesAvailableForSaleAccumulatedGrossUnrealizedGainBeforeTax_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_EquitySecuritiesAvailableForSaleAccumulatedGrossUnrealizedLossBeforeTax" xlink:type="locator" xlink:label="pbhc_EquitySecuritiesAvailableForSaleAccumulatedGrossUnrealizedLossBeforeTax" />
    <link:label xml:lang="en-US" xlink:label="pbhc_EquitySecuritiesAvailableForSaleAccumulatedGrossUnrealizedLossBeforeTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Equity securities, available-for-sale, accumulated gross unrealized loss, before tax.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_EquitySecuritiesAvailableForSaleAccumulatedGrossUnrealizedLossBeforeTax" xlink:to="pbhc_EquitySecuritiesAvailableForSaleAccumulatedGrossUnrealizedLossBeforeTax_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_AvailableForSaleDebtAndEquitySecuritiesAccumulatedGrossUnrealizedGainBeforeTax" xlink:type="locator" xlink:label="pbhc_AvailableForSaleDebtAndEquitySecuritiesAccumulatedGrossUnrealizedGainBeforeTax" />
    <link:label xml:lang="en-US" xlink:label="pbhc_AvailableForSaleDebtAndEquitySecuritiesAccumulatedGrossUnrealizedGainBeforeTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Available for sale debt and equity securities accumulated gross unrealized gain before tax.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_AvailableForSaleDebtAndEquitySecuritiesAccumulatedGrossUnrealizedGainBeforeTax" xlink:to="pbhc_AvailableForSaleDebtAndEquitySecuritiesAccumulatedGrossUnrealizedGainBeforeTax_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_AvailableForSaleDebtAndEquitySecuritiesAccumulatedGrossUnrealizedLossBeforeTax" xlink:type="locator" xlink:label="pbhc_AvailableForSaleDebtAndEquitySecuritiesAccumulatedGrossUnrealizedLossBeforeTax" />
    <link:label xml:lang="en-US" xlink:label="pbhc_AvailableForSaleDebtAndEquitySecuritiesAccumulatedGrossUnrealizedLossBeforeTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Available for sale debt and equity securities accumulated gross unrealized loss before tax.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_AvailableForSaleDebtAndEquitySecuritiesAccumulatedGrossUnrealizedLossBeforeTax" xlink:to="pbhc_AvailableForSaleDebtAndEquitySecuritiesAccumulatedGrossUnrealizedLossBeforeTax_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_AvailableForSaleDebtAndEquitySecurities" xlink:type="locator" xlink:label="pbhc_AvailableForSaleDebtAndEquitySecurities" />
    <link:label xml:lang="en-US" xlink:label="pbhc_AvailableForSaleDebtAndEquitySecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Available for sale debt and equity securities.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_AvailableForSaleDebtAndEquitySecurities" xlink:to="pbhc_AvailableForSaleDebtAndEquitySecurities_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_HeldToMaturityNumberOfSecuritiesInUnrealizedLossPositionsLessThanTwelveMonths" xlink:type="locator" xlink:label="pbhc_HeldToMaturityNumberOfSecuritiesInUnrealizedLossPositionsLessThanTwelveMonths" />
    <link:label xml:lang="en-US" xlink:label="pbhc_HeldToMaturityNumberOfSecuritiesInUnrealizedLossPositionsLessThanTwelveMonths_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">This element represent the number of held to maturity securities in unrealized loss positions for less than twelve months.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_HeldToMaturityNumberOfSecuritiesInUnrealizedLossPositionsLessThanTwelveMonths" xlink:to="pbhc_HeldToMaturityNumberOfSecuritiesInUnrealizedLossPositionsLessThanTwelveMonths_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_HeldToMaturityNumberOfSecuritiesInUnrealizedLossPositionsTwelveMonthsOrMore" xlink:type="locator" xlink:label="pbhc_HeldToMaturityNumberOfSecuritiesInUnrealizedLossPositionsTwelveMonthsOrMore" />
    <link:label xml:lang="en-US" xlink:label="pbhc_HeldToMaturityNumberOfSecuritiesInUnrealizedLossPositionsTwelveMonthsOrMore_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">This element represent the number of held to maturity securities in unrealized loss positions for twelve months or more.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_HeldToMaturityNumberOfSecuritiesInUnrealizedLossPositionsTwelveMonthsOrMore" xlink:to="pbhc_HeldToMaturityNumberOfSecuritiesInUnrealizedLossPositionsTwelveMonthsOrMore_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_FinancingReceivableAllowanceForCreditLossesSpecificallyReserved" xlink:type="locator" xlink:label="pbhc_FinancingReceivableAllowanceForCreditLossesSpecificallyReserved" />
    <link:label xml:lang="en-US" xlink:label="pbhc_FinancingReceivableAllowanceForCreditLossesSpecificallyReserved_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Amount of financing receivables in allowance to specifically reserved.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_FinancingReceivableAllowanceForCreditLossesSpecificallyReserved" xlink:to="pbhc_FinancingReceivableAllowanceForCreditLossesSpecificallyReserved_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_FinancingReceivableAllowanceForCreditLossesHistoricalLossRate" xlink:type="locator" xlink:label="pbhc_FinancingReceivableAllowanceForCreditLossesHistoricalLossRate" />
    <link:label xml:lang="en-US" xlink:label="pbhc_FinancingReceivableAllowanceForCreditLossesHistoricalLossRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Amount of financing receivables in allowance to historical loss rate.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_FinancingReceivableAllowanceForCreditLossesHistoricalLossRate" xlink:to="pbhc_FinancingReceivableAllowanceForCreditLossesHistoricalLossRate_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_FinancingReceivableAllowanceForCreditLossesQualitativeFactors" xlink:type="locator" xlink:label="pbhc_FinancingReceivableAllowanceForCreditLossesQualitativeFactors" />
    <link:label xml:lang="en-US" xlink:label="pbhc_FinancingReceivableAllowanceForCreditLossesQualitativeFactors_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Amount of financing receivables in allowance to qualitative factors.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_FinancingReceivableAllowanceForCreditLossesQualitativeFactors" xlink:to="pbhc_FinancingReceivableAllowanceForCreditLossesQualitativeFactors_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_FinancingReceivableAllowanceForCreditLossesOther" xlink:type="locator" xlink:label="pbhc_FinancingReceivableAllowanceForCreditLossesOther" />
    <link:label xml:lang="en-US" xlink:label="pbhc_FinancingReceivableAllowanceForCreditLossesOther_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Financing receivable allowance for credit losses other.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_FinancingReceivableAllowanceForCreditLossesOther" xlink:to="pbhc_FinancingReceivableAllowanceForCreditLossesOther_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_TimeDeposits250000OrMore" xlink:type="locator" xlink:label="pbhc_TimeDeposits250000OrMore" />
    <link:label xml:lang="en-US" xlink:label="pbhc_TimeDeposits250000OrMore_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The aggregate amount of domestic and foreign time deposit liabilities (including certificates of deposits, individual retirement accounts and open accounts) in denominations of $250,000 or more.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_TimeDeposits250000OrMore" xlink:to="pbhc_TimeDeposits250000OrMore_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_DepositsMoneyManagementAccounts" xlink:type="locator" xlink:label="pbhc_DepositsMoneyManagementAccounts" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DepositsMoneyManagementAccounts_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Amount of deposits in accounts that offer many of the same services as checking accounts although transactions may be somewhat more limited, share some of the characteristics of a money market fund and are insured by the Federal government, also known as money management accounts.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_DepositsMoneyManagementAccounts" xlink:to="pbhc_DepositsMoneyManagementAccounts_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_MortgageEscrowFunds" xlink:type="locator" xlink:label="pbhc_MortgageEscrowFunds" />
    <link:label xml:lang="en-US" xlink:label="pbhc_MortgageEscrowFunds_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Mortgage funds held in escrow.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_MortgageEscrowFunds" xlink:to="pbhc_MortgageEscrowFunds_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_LongTermDebtMaturitiesSummaryDueInWithinFiveYears" xlink:type="locator" xlink:label="pbhc_LongTermDebtMaturitiesSummaryDueInWithinFiveYears" />
    <link:label xml:lang="en-US" xlink:label="pbhc_LongTermDebtMaturitiesSummaryDueInWithinFiveYears_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Amount of long term debt maturing within five years following the latest fiscal year. Excludes interim and annual periods when interim periods are reported on a rolling approach, from latest balance sheet date.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_LongTermDebtMaturitiesSummaryDueInWithinFiveYears" xlink:to="pbhc_LongTermDebtMaturitiesSummaryDueInWithinFiveYears_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_LongTermDebtMaturitiesSummaryDueInWithinTenYears" xlink:type="locator" xlink:label="pbhc_LongTermDebtMaturitiesSummaryDueInWithinTenYears" />
    <link:label xml:lang="en-US" xlink:label="pbhc_LongTermDebtMaturitiesSummaryDueInWithinTenYears_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Amount of long term debt maturing within ten years following the latest fiscal year. Excludes interim and annual periods when interim periods are reported on a rolling approach, from latest balance sheet date.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_LongTermDebtMaturitiesSummaryDueInWithinTenYears" xlink:to="pbhc_LongTermDebtMaturitiesSummaryDueInWithinTenYears_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_LongTermDebtMaturitiesSummaryDueInWithinSeventeenYears" xlink:type="locator" xlink:label="pbhc_LongTermDebtMaturitiesSummaryDueInWithinSeventeenYears" />
    <link:label xml:lang="en-US" xlink:label="pbhc_LongTermDebtMaturitiesSummaryDueInWithinSeventeenYears_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Amount of long term debt maturing within seventeen years following the latest fiscal year. Excludes interim and annual periods when interim periods are reported on a rolling approach, from latest balance sheet date.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_LongTermDebtMaturitiesSummaryDueInWithinSeventeenYears" xlink:to="pbhc_LongTermDebtMaturitiesSummaryDueInWithinSeventeenYears_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_UnrealizedLossAndTransitionObligationForPensionAndOtherPostretirementObligations" xlink:type="locator" xlink:label="pbhc_UnrealizedLossAndTransitionObligationForPensionAndOtherPostretirementObligations" />
    <link:label xml:lang="en-US" xlink:label="pbhc_UnrealizedLossAndTransitionObligationForPensionAndOtherPostretirementObligations_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The total of unrealized loss and transition, before tax, for pension and other postretirement obligations included in accumulated other comprehensive income associated with a defined benefit pension or other postretirement plan(s)because they have yet to be recognized as components of net periodic benefit cost.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_UnrealizedLossAndTransitionObligationForPensionAndOtherPostretirementObligations" xlink:to="pbhc_UnrealizedLossAndTransitionObligationForPensionAndOtherPostretirementObligations_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_UnrealizedLossAndTransitionObligationForPensionAndOtherPostretirementObligationsTaxEffect" xlink:type="locator" xlink:label="pbhc_UnrealizedLossAndTransitionObligationForPensionAndOtherPostretirementObligationsTaxEffect" />
    <link:label xml:lang="en-US" xlink:label="pbhc_UnrealizedLossAndTransitionObligationForPensionAndOtherPostretirementObligationsTaxEffect_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Tax effect of unrealized loss and transition obligation for pension and other postretirement obligation included in accumulated other comprehensive income (loss).</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_UnrealizedLossAndTransitionObligationForPensionAndOtherPostretirementObligationsTaxEffect" xlink:to="pbhc_UnrealizedLossAndTransitionObligationForPensionAndOtherPostretirementObligationsTaxEffect_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_DefinedBenefitPlanAccumulatedOtherComprehensiveIncomeLossTax" xlink:type="locator" xlink:label="pbhc_DefinedBenefitPlanAccumulatedOtherComprehensiveIncomeLossTax" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DefinedBenefitPlanAccumulatedOtherComprehensiveIncomeLossTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The tax effect of net gain (loss), prior service cost (credit), and transition assets (obligations), as well as minimum pension liability if still remaining, included in accumulated other comprehensive income associated with a defined benefit pension or other postretirement plan(s).</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_DefinedBenefitPlanAccumulatedOtherComprehensiveIncomeLossTax" xlink:to="pbhc_DefinedBenefitPlanAccumulatedOtherComprehensiveIncomeLossTax_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_MutualFundsEquityLargeCapValueMember" xlink:type="locator" xlink:label="pbhc_MutualFundsEquityLargeCapValueMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_MutualFundsEquityLargeCapValueMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">An investment that pools funds from many investors to invest in a combination of underlying investments, primarily equity investments.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_MutualFundsEquityLargeCapValueMember" xlink:to="pbhc_MutualFundsEquityLargeCapValueMember_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_MutualFundsEquityLargeCapGrowthMember" xlink:type="locator" xlink:label="pbhc_MutualFundsEquityLargeCapGrowthMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_MutualFundsEquityLargeCapGrowthMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">An investment that pools funds from many investors to invest in a combination of underlying investments, primarily equity investments.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_MutualFundsEquityLargeCapGrowthMember" xlink:to="pbhc_MutualFundsEquityLargeCapGrowthMember_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_MutualFundsEquityLargeCapCoreMember" xlink:type="locator" xlink:label="pbhc_MutualFundsEquityLargeCapCoreMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_MutualFundsEquityLargeCapCoreMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">An investment that pools funds from many investors to invest in a combination of underlying investments, primarily equity investments.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_MutualFundsEquityLargeCapCoreMember" xlink:to="pbhc_MutualFundsEquityLargeCapCoreMember_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_MutualFundsEquityMidCapValueMember" xlink:type="locator" xlink:label="pbhc_MutualFundsEquityMidCapValueMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_MutualFundsEquityMidCapValueMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">An investment that pools funds from many investors to invest in a combination of underlying investments, primarily equity investments.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_MutualFundsEquityMidCapValueMember" xlink:to="pbhc_MutualFundsEquityMidCapValueMember_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_MutualFundsEquityMidCapGrowthMember" xlink:type="locator" xlink:label="pbhc_MutualFundsEquityMidCapGrowthMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_MutualFundsEquityMidCapGrowthMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">An investment that pools funds from many investors to invest in a combination of underlying investments, primarily equity investments.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_MutualFundsEquityMidCapGrowthMember" xlink:to="pbhc_MutualFundsEquityMidCapGrowthMember_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_MutualFundsEquityMidCapCoreMember" xlink:type="locator" xlink:label="pbhc_MutualFundsEquityMidCapCoreMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_MutualFundsEquityMidCapCoreMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">An investment that pools funds from many investors to invest in a combination of underlying investments, primarily equity investments.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_MutualFundsEquityMidCapCoreMember" xlink:to="pbhc_MutualFundsEquityMidCapCoreMember_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_MutualFundsEquitySmallCapValueMember" xlink:type="locator" xlink:label="pbhc_MutualFundsEquitySmallCapValueMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_MutualFundsEquitySmallCapValueMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">An investment that pools funds from many investors to invest in a combination of underlying investments, primarily equity investments.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_MutualFundsEquitySmallCapValueMember" xlink:to="pbhc_MutualFundsEquitySmallCapValueMember_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_MutualFundsEquitySmallCapGrowthMember" xlink:type="locator" xlink:label="pbhc_MutualFundsEquitySmallCapGrowthMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_MutualFundsEquitySmallCapGrowthMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">An investment that pools funds from many investors to invest in a combination of underlying investments, primarily equity investments.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_MutualFundsEquitySmallCapGrowthMember" xlink:to="pbhc_MutualFundsEquitySmallCapGrowthMember_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_MutualFundsEquitySmallCapCoreMember" xlink:type="locator" xlink:label="pbhc_MutualFundsEquitySmallCapCoreMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_MutualFundsEquitySmallCapCoreMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">An investment that pools funds from many investors to invest in a combination of underlying investments, primarily equity investments.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_MutualFundsEquitySmallCapCoreMember" xlink:to="pbhc_MutualFundsEquitySmallCapCoreMember_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_InternationalEquityMember" xlink:type="locator" xlink:label="pbhc_InternationalEquityMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_InternationalEquityMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">An investment that pools funds from many investors to invest in a combination of underlying investments, primarily equity investments.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_InternationalEquityMember" xlink:to="pbhc_InternationalEquityMember_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_MutualFundsFixedIncomeUSCoreMember" xlink:type="locator" xlink:label="pbhc_MutualFundsFixedIncomeUSCoreMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_MutualFundsFixedIncomeUSCoreMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Investment that pools funds from investors to invest in a combination of underlying investments, primarily fixed income investments intermediately.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_MutualFundsFixedIncomeUSCoreMember" xlink:to="pbhc_MutualFundsFixedIncomeUSCoreMember_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_MutualFundsFixedIncomeIntermediateDurationMember" xlink:type="locator" xlink:label="pbhc_MutualFundsFixedIncomeIntermediateDurationMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_MutualFundsFixedIncomeIntermediateDurationMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Investment that pools funds from investors to invest in a combination of underlying investments, primarily fixed income investments intermediately.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_MutualFundsFixedIncomeIntermediateDurationMember" xlink:to="pbhc_MutualFundsFixedIncomeIntermediateDurationMember_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_MutualFundsFixedIncomeLongDurationMember" xlink:type="locator" xlink:label="pbhc_MutualFundsFixedIncomeLongDurationMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_MutualFundsFixedIncomeLongDurationMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Investment that pools funds from investors to invest in a combination of underlying investments, primarily fixed income investments intermediately.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_MutualFundsFixedIncomeLongDurationMember" xlink:to="pbhc_MutualFundsFixedIncomeLongDurationMember_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_DeferredTaxAssetsSubordinatedLoanInterest" xlink:type="locator" xlink:label="pbhc_DeferredTaxAssetsSubordinatedLoanInterest" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DeferredTaxAssetsSubordinatedLoanInterest_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Amount before allocation of valuation allowances of deferred tax asset attributable to deductible temporary differences from subordinated loan interest.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_DeferredTaxAssetsSubordinatedLoanInterest" xlink:to="pbhc_DeferredTaxAssetsSubordinatedLoanInterest_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_DeferredTaxAssetsLoanOriginationFees" xlink:type="locator" xlink:label="pbhc_DeferredTaxAssetsLoanOriginationFees" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DeferredTaxAssetsLoanOriginationFees_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Amount before allocation of valuation allowances of deferred tax asset attributable to taxable temporary differences related to loan origination fees.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_DeferredTaxAssetsLoanOriginationFees" xlink:to="pbhc_DeferredTaxAssetsLoanOriginationFees_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_DeferredTaxAssetsHeldToMaturitySecurities" xlink:type="locator" xlink:label="pbhc_DeferredTaxAssetsHeldToMaturitySecurities" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DeferredTaxAssetsHeldToMaturitySecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Amount before allocation of valuation allowances of deferred tax asset attributable to held-to-maturity securities.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_DeferredTaxAssetsHeldToMaturitySecurities" xlink:to="pbhc_DeferredTaxAssetsHeldToMaturitySecurities_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_DeferredTaxAssetsCommunityServiceActivities" xlink:type="locator" xlink:label="pbhc_DeferredTaxAssetsCommunityServiceActivities" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DeferredTaxAssetsCommunityServiceActivities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Deferred tax assets community service activities.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_DeferredTaxAssetsCommunityServiceActivities" xlink:to="pbhc_DeferredTaxAssetsCommunityServiceActivities_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_DeferredTaxLiabilitiesPrepaidPension" xlink:type="locator" xlink:label="pbhc_DeferredTaxLiabilitiesPrepaidPension" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DeferredTaxLiabilitiesPrepaidPension_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Amount of deferred tax consequences attributable to taxable temporary differences derived from prepaid pension.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_DeferredTaxLiabilitiesPrepaidPension" xlink:to="pbhc_DeferredTaxLiabilitiesPrepaidPension_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_DeferredTaxLiabilitiesAccretion" xlink:type="locator" xlink:label="pbhc_DeferredTaxLiabilitiesAccretion" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DeferredTaxLiabilitiesAccretion_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Amount of deferred tax liability attributable to taxable temporary differences related to accretion.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_DeferredTaxLiabilitiesAccretion" xlink:to="pbhc_DeferredTaxLiabilitiesAccretion_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_DeferredTaxLiabilitiesLoanOriginationFees" xlink:type="locator" xlink:label="pbhc_DeferredTaxLiabilitiesLoanOriginationFees" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DeferredTaxLiabilitiesLoanOriginationFees_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Deferred tax liabilities loan origination fees.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_DeferredTaxLiabilitiesLoanOriginationFees" xlink:to="pbhc_DeferredTaxLiabilitiesLoanOriginationFees_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_UsTreasuryAgenciesAndGsesFairValueDisclosure" xlink:type="locator" xlink:label="pbhc_UsTreasuryAgenciesAndGsesFairValueDisclosure" />
    <link:label xml:lang="en-US" xlink:label="pbhc_UsTreasuryAgenciesAndGsesFairValueDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">This element represents the portion of the balance sheet assertion valued at fair value by the entity whether such amount is presented as a separate caption or as a parenthetical disclosure. Additionally, this element may be used in connection with the fair value disclosures required in the footnote disclosures to the financial statements. The element may be used in both the balance sheet and disclosure in the same submission. This item represents the amount shown on an entity&apos;s books for investments in US Treasury, agencies and GSEs debt investment securities.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_UsTreasuryAgenciesAndGsesFairValueDisclosure" xlink:to="pbhc_UsTreasuryAgenciesAndGsesFairValueDisclosure_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_StateAndPoliticalSubdivisionsFairValueDisclosure" xlink:type="locator" xlink:label="pbhc_StateAndPoliticalSubdivisionsFairValueDisclosure" />
    <link:label xml:lang="en-US" xlink:label="pbhc_StateAndPoliticalSubdivisionsFairValueDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">This element represents the portion of the balance sheet assertion valued at fair value by the entity whether such amount is presented as a separate caption or as a parenthetical disclosure. Additionally, this element may be used in connection with the fair value disclosures required in the footnote disclosures to the financial statements. The element may be used in both the balance sheet and disclosure in the same submission. This item represents the amount shown on an entity&apos;s books for investments in State and political subdivisions debt investment securities.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_StateAndPoliticalSubdivisionsFairValueDisclosure" xlink:to="pbhc_StateAndPoliticalSubdivisionsFairValueDisclosure_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_CorporateFairValueDisclosure" xlink:type="locator" xlink:label="pbhc_CorporateFairValueDisclosure" />
    <link:label xml:lang="en-US" xlink:label="pbhc_CorporateFairValueDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">This element represents the portion of the balance sheet assertion valued at fair value by the entity whether such amount is presented as a separate caption or as a parenthetical disclosure. Additionally, this element may be used in connection with the fair value disclosures required in the footnote disclosures to the financial statements. The element may be used in both the balance sheet and disclosure in the same submission. This item represents the amount shown on an entity&apos;s books for investments in Corporate debt investment securities.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_CorporateFairValueDisclosure" xlink:to="pbhc_CorporateFairValueDisclosure_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_AssetBackedSecuritiesDisclosure" xlink:type="locator" xlink:label="pbhc_AssetBackedSecuritiesDisclosure" />
    <link:label xml:lang="en-US" xlink:label="pbhc_AssetBackedSecuritiesDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">This element represents the portion of the balance sheet assertion valued at fair value by the entity whether such amount is presented as a separate caption or as a parenthetical disclosure. Additionally, this element may be used in connection with the fair value disclosures required in the footnote disclosures to the financial statements. The element may be used in both the balance sheet and disclosure in the same submission. This item represents the amount shown on an entity&apos;s books for investments in Asset backed securities investment securities.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_AssetBackedSecuritiesDisclosure" xlink:to="pbhc_AssetBackedSecuritiesDisclosure_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ResidentialMortgageBackedAgencyFairValueDisclosure" xlink:type="locator" xlink:label="pbhc_ResidentialMortgageBackedAgencyFairValueDisclosure" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ResidentialMortgageBackedAgencyFairValueDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">This element represents the portion of the balance sheet assertion valued at fair value by the entity whether such amount is presented as a separate caption or as a parenthetical disclosure. Additionally, this element may be used in connection with the fair value disclosures required in the footnote disclosures to the financial statements. The element may be used in both the balance sheet and disclosure in the same submission. This item represents the amount shown on an entity&apos;s books for investments in residential mortgage backed agencies debt investment securities.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ResidentialMortgageBackedAgencyFairValueDisclosure" xlink:to="pbhc_ResidentialMortgageBackedAgencyFairValueDisclosure_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_CollateralizedMortgageObligationsAgencyFairValueDisclosure" xlink:type="locator" xlink:label="pbhc_CollateralizedMortgageObligationsAgencyFairValueDisclosure" />
    <link:label xml:lang="en-US" xlink:label="pbhc_CollateralizedMortgageObligationsAgencyFairValueDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">This element represents the portion of the balance sheet assertion valued at fair value by the entity whether such amount is presented as a separate caption or as a parenthetical disclosure. Additionally, this element may be used in connection with the fair value disclosures required in the footnote disclosures to the financial statements. The element may be used in both the balance sheet and disclosure in the same submission. This item represents the amount shown on an entity&apos;s books for investments in collateralized mortgage obligations agencies debt investment securities.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_CollateralizedMortgageObligationsAgencyFairValueDisclosure" xlink:to="pbhc_CollateralizedMortgageObligationsAgencyFairValueDisclosure_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_CollateralizedMortgageObligationsPrivateLabelFairValueDisclosure" xlink:type="locator" xlink:label="pbhc_CollateralizedMortgageObligationsPrivateLabelFairValueDisclosure" />
    <link:label xml:lang="en-US" xlink:label="pbhc_CollateralizedMortgageObligationsPrivateLabelFairValueDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">This element represents the portion of the balance sheet assertion valued at fair value by the entity whether such amount is presented as a separate caption or as a parenthetical disclosure. Additionally, this element may be used in connection with the fair value disclosures required in the footnote disclosures to the financial statements. The element may be used in both the balance sheet and disclosure in the same submission. This item represents the amount shown on an entity&apos;s books for investments in collateralized mortgage obligations private label investment securities.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_CollateralizedMortgageObligationsPrivateLabelFairValueDisclosure" xlink:to="pbhc_CollateralizedMortgageObligationsPrivateLabelFairValueDisclosure_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_IncomeLossBeforeEquityInUndistributedEarningsOfSubsidiaries" xlink:type="locator" xlink:label="pbhc_IncomeLossBeforeEquityInUndistributedEarningsOfSubsidiaries" />
    <link:label xml:lang="en-US" xlink:label="pbhc_IncomeLossBeforeEquityInUndistributedEarningsOfSubsidiaries_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">This element represents the income or loss from continuing operations attributable to the economic entity which may also be defined as revenue less expenses and taxes from ongoing operations before extraordinary items, before equity in undistributed earnings of subsidiaries.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_IncomeLossBeforeEquityInUndistributedEarningsOfSubsidiaries" xlink:to="pbhc_IncomeLossBeforeEquityInUndistributedEarningsOfSubsidiaries_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_EquityInUndistributedEarningsOfSubsidiaries" xlink:type="locator" xlink:label="pbhc_EquityInUndistributedEarningsOfSubsidiaries" />
    <link:label xml:lang="en-US" xlink:label="pbhc_EquityInUndistributedEarningsOfSubsidiaries_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Equity in of undistributed earnings of subsidiaries.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_EquityInUndistributedEarningsOfSubsidiaries" xlink:to="pbhc_EquityInUndistributedEarningsOfSubsidiaries_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_AmortizationOfOperatingLeases" xlink:type="locator" xlink:label="pbhc_AmortizationOfOperatingLeases" />
    <link:label xml:lang="en-US" xlink:label="pbhc_AmortizationOfOperatingLeases_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Amortization of operating leases.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_AmortizationOfOperatingLeases" xlink:to="pbhc_AmortizationOfOperatingLeases_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_GainsLossesOnRealEstateAcquiredThroughForeclosure" xlink:type="locator" xlink:label="pbhc_GainsLossesOnRealEstateAcquiredThroughForeclosure" />
    <link:label xml:lang="en-US" xlink:label="pbhc_GainsLossesOnRealEstateAcquiredThroughForeclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Gains losses on real estate acquired through foreclosure.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_GainsLossesOnRealEstateAcquiredThroughForeclosure" xlink:to="pbhc_GainsLossesOnRealEstateAcquiredThroughForeclosure_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_PaymentsForProceedsFromSaleOfAvailableForSaleSecuritiesDebt" xlink:type="locator" xlink:label="pbhc_PaymentsForProceedsFromSaleOfAvailableForSaleSecuritiesDebt" />
    <link:label xml:lang="en-US" xlink:label="pbhc_PaymentsForProceedsFromSaleOfAvailableForSaleSecuritiesDebt_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Payments for (proceeds from) sale of available for sale securities debt.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_PaymentsForProceedsFromSaleOfAvailableForSaleSecuritiesDebt" xlink:to="pbhc_PaymentsForProceedsFromSaleOfAvailableForSaleSecuritiesDebt_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_AmortizationAccretionOfMortgageServicingRights" xlink:type="locator" xlink:label="pbhc_AmortizationAccretionOfMortgageServicingRights" />
    <link:label xml:lang="en-US" xlink:label="pbhc_AmortizationAccretionOfMortgageServicingRights_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Amortization (accretion) of mortgage servicing rights.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_AmortizationAccretionOfMortgageServicingRights" xlink:to="pbhc_AmortizationAccretionOfMortgageServicingRights_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_AmortizationAccretionOfDeferredLoanCosts" xlink:type="locator" xlink:label="pbhc_AmortizationAccretionOfDeferredLoanCosts" />
    <link:label xml:lang="en-US" xlink:label="pbhc_AmortizationAccretionOfDeferredLoanCosts_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Amortization accretion of deferred loan costs.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_AmortizationAccretionOfDeferredLoanCosts" xlink:to="pbhc_AmortizationAccretionOfDeferredLoanCosts_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_AmortizationOfFinancingCostFromSubordinatedDebt" xlink:type="locator" xlink:label="pbhc_AmortizationOfFinancingCostFromSubordinatedDebt" />
    <link:label xml:lang="en-US" xlink:label="pbhc_AmortizationOfFinancingCostFromSubordinatedDebt_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Amortization of financing cost from subordinated debt.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_AmortizationOfFinancingCostFromSubordinatedDebt" xlink:to="pbhc_AmortizationOfFinancingCostFromSubordinatedDebt_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_BankOwnedLifeInsuranceIncomeOperatingActivities" xlink:type="locator" xlink:label="pbhc_BankOwnedLifeInsuranceIncomeOperatingActivities" />
    <link:label xml:lang="en-US" xlink:label="pbhc_BankOwnedLifeInsuranceIncomeOperatingActivities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Operating activities earnings on or other increases in the value of the cash surrender value of bank owned life insurance policies.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_BankOwnedLifeInsuranceIncomeOperatingActivities" xlink:to="pbhc_BankOwnedLifeInsuranceIncomeOperatingActivities_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_PaymentsToAcquireAvailableForSaleDebtAndEquitySecurities" xlink:type="locator" xlink:label="pbhc_PaymentsToAcquireAvailableForSaleDebtAndEquitySecurities" />
    <link:label xml:lang="en-US" xlink:label="pbhc_PaymentsToAcquireAvailableForSaleDebtAndEquitySecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Payments to acquire available for sale debt and equity securities.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_PaymentsToAcquireAvailableForSaleDebtAndEquitySecurities" xlink:to="pbhc_PaymentsToAcquireAvailableForSaleDebtAndEquitySecurities_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ProceedsFromSaleOfAvailableForSaleDebtAndEquitySecurities" xlink:type="locator" xlink:label="pbhc_ProceedsFromSaleOfAvailableForSaleDebtAndEquitySecurities" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ProceedsFromSaleOfAvailableForSaleDebtAndEquitySecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Proceeds from sale of available for sale debt and equity securities.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ProceedsFromSaleOfAvailableForSaleDebtAndEquitySecurities" xlink:to="pbhc_ProceedsFromSaleOfAvailableForSaleDebtAndEquitySecurities_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_IncreaseDecreaseInBrokeredDeposits" xlink:type="locator" xlink:label="pbhc_IncreaseDecreaseInBrokeredDeposits" />
    <link:label xml:lang="en-US" xlink:label="pbhc_IncreaseDecreaseInBrokeredDeposits_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Refers to Net cash inflow (outflow) of brokered deposits.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_IncreaseDecreaseInBrokeredDeposits" xlink:to="pbhc_IncreaseDecreaseInBrokeredDeposits_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ProceedsFromJuniorSubordinatedDebentureOwedToUnconsolidatedSubsidiaryTrust" xlink:type="locator" xlink:label="pbhc_ProceedsFromJuniorSubordinatedDebentureOwedToUnconsolidatedSubsidiaryTrust" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ProceedsFromJuniorSubordinatedDebentureOwedToUnconsolidatedSubsidiaryTrust_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Proceeds from junior subordinated debenture owed to unconsolidated subsidiary trust.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ProceedsFromJuniorSubordinatedDebentureOwedToUnconsolidatedSubsidiaryTrust" xlink:to="pbhc_ProceedsFromJuniorSubordinatedDebentureOwedToUnconsolidatedSubsidiaryTrust_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_PaymentsOfDividendsWarrants" xlink:type="locator" xlink:label="pbhc_PaymentsOfDividendsWarrants" />
    <link:label xml:lang="en-US" xlink:label="pbhc_PaymentsOfDividendsWarrants_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Payments of dividends warrants.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_PaymentsOfDividendsWarrants" xlink:to="pbhc_PaymentsOfDividendsWarrants_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ProceedsFromFinanceLeaseTransaction" xlink:type="locator" xlink:label="pbhc_ProceedsFromFinanceLeaseTransaction" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ProceedsFromFinanceLeaseTransaction_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Proceeds from finance lease transaction.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ProceedsFromFinanceLeaseTransaction" xlink:to="pbhc_ProceedsFromFinanceLeaseTransaction_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_NetGainsLossesOnSalesOfLoansAndForeclosedRealEstate" xlink:type="locator" xlink:label="pbhc_NetGainsLossesOnSalesOfLoansAndForeclosedRealEstate" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NetGainsLossesOnSalesOfLoansAndForeclosedRealEstate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The net gain/loss resulting from a sale of loans, including adjustments to record loans classified as held-for-sale at the lower-of-cost-or-market and fair value adjustments to loan held for investment purposes and the difference between the carrying value and the sale price of real estate. This element refers to the gain/loss included in earnings and not to the cash proceeds of the sale.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_NetGainsLossesOnSalesOfLoansAndForeclosedRealEstate" xlink:to="pbhc_NetGainsLossesOnSalesOfLoansAndForeclosedRealEstate_lbl" />
    <link:loc xlink:href="https://xbrl.sec.gov/dei/2020/dei-2020-01-31.xsd#dei_CoverAbstract" xlink:type="locator" xlink:label="dei_CoverAbstract" />
    <link:label xml:lang="en-US" xlink:label="dei_CoverAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Cover [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="dei_CoverAbstract" xlink:to="dei_CoverAbstract_lbl" />
    <link:loc xlink:href="https://xbrl.sec.gov/dei/2020/dei-2020-01-31.xsd#dei_DocumentType" xlink:type="locator" xlink:label="dei_DocumentType" />
    <link:label xml:lang="en-US" xlink:label="dei_DocumentType_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Document Type</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="dei_DocumentType" xlink:to="dei_DocumentType_lbl" />
    <link:label xml:lang="en-US" xlink:label="dei_DocumentType_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Document Type</link:label>
    <link:loc xlink:href="https://xbrl.sec.gov/dei/2020/dei-2020-01-31.xsd#dei_AmendmentFlag" xlink:type="locator" xlink:label="dei_AmendmentFlag" />
    <link:label xml:lang="en-US" xlink:label="dei_AmendmentFlag_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Amendment Flag</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="dei_AmendmentFlag" xlink:to="dei_AmendmentFlag_lbl" />
    <link:label xml:lang="en-US" xlink:label="dei_AmendmentFlag_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Amendment Flag</link:label>
    <link:loc xlink:href="https://xbrl.sec.gov/dei/2020/dei-2020-01-31.xsd#dei_DocumentPeriodEndDate" xlink:type="locator" xlink:label="dei_DocumentPeriodEndDate" />
    <link:label xml:lang="en-US" xlink:label="dei_DocumentPeriodEndDate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Document Period End Date</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="dei_DocumentPeriodEndDate" xlink:to="dei_DocumentPeriodEndDate_lbl" />
    <link:label xml:lang="en-US" xlink:label="dei_DocumentPeriodEndDate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Document Period End Date</link:label>
    <link:loc xlink:href="https://xbrl.sec.gov/dei/2020/dei-2020-01-31.xsd#dei_DocumentFiscalYearFocus" xlink:type="locator" xlink:label="dei_DocumentFiscalYearFocus" />
    <link:label xml:lang="en-US" xlink:label="dei_DocumentFiscalYearFocus_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Document Fiscal Year Focus</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="dei_DocumentFiscalYearFocus" xlink:to="dei_DocumentFiscalYearFocus_lbl" />
    <link:label xml:lang="en-US" xlink:label="dei_DocumentFiscalYearFocus_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Document Fiscal Year Focus</link:label>
    <link:loc xlink:href="https://xbrl.sec.gov/dei/2020/dei-2020-01-31.xsd#dei_DocumentFiscalPeriodFocus" xlink:type="locator" xlink:label="dei_DocumentFiscalPeriodFocus" />
    <link:label xml:lang="en-US" xlink:label="dei_DocumentFiscalPeriodFocus_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Document Fiscal Period Focus</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="dei_DocumentFiscalPeriodFocus" xlink:to="dei_DocumentFiscalPeriodFocus_lbl" />
    <link:label xml:lang="en-US" xlink:label="dei_DocumentFiscalPeriodFocus_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Document Fiscal Period Focus</link:label>
    <link:loc xlink:href="https://xbrl.sec.gov/dei/2020/dei-2020-01-31.xsd#dei_EntityRegistrantName" xlink:type="locator" xlink:label="dei_EntityRegistrantName" />
    <link:label xml:lang="en-US" xlink:label="dei_EntityRegistrantName_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Entity Registrant Name</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="dei_EntityRegistrantName" xlink:to="dei_EntityRegistrantName_lbl" />
    <link:label xml:lang="en-US" xlink:label="dei_EntityRegistrantName_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Entity Registrant Name</link:label>
    <link:label xml:lang="en-US" xlink:label="dei_EntityCentralIndexKey_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Entity Central Index Key</link:label>
    <link:loc xlink:href="https://xbrl.sec.gov/dei/2020/dei-2020-01-31.xsd#dei_EntityCurrentReportingStatus" xlink:type="locator" xlink:label="dei_EntityCurrentReportingStatus" />
    <link:label xml:lang="en-US" xlink:label="dei_EntityCurrentReportingStatus_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Entity Current Reporting Status</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="dei_EntityCurrentReportingStatus" xlink:to="dei_EntityCurrentReportingStatus_lbl" />
    <link:label xml:lang="en-US" xlink:label="dei_EntityCurrentReportingStatus_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Entity Current Reporting Status</link:label>
    <link:loc xlink:href="https://xbrl.sec.gov/dei/2020/dei-2020-01-31.xsd#dei_EntityVoluntaryFilers" xlink:type="locator" xlink:label="dei_EntityVoluntaryFilers" />
    <link:label xml:lang="en-US" xlink:label="dei_EntityVoluntaryFilers_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Entity Voluntary Filers</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="dei_EntityVoluntaryFilers" xlink:to="dei_EntityVoluntaryFilers_lbl" />
    <link:label xml:lang="en-US" xlink:label="dei_EntityVoluntaryFilers_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Entity Voluntary Filers</link:label>
    <link:loc xlink:href="https://xbrl.sec.gov/dei/2020/dei-2020-01-31.xsd#dei_EntityInteractiveDataCurrent" xlink:type="locator" xlink:label="dei_EntityInteractiveDataCurrent" />
    <link:label xml:lang="en-US" xlink:label="dei_EntityInteractiveDataCurrent_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Entity Interactive Data Current</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="dei_EntityInteractiveDataCurrent" xlink:to="dei_EntityInteractiveDataCurrent_lbl" />
    <link:label xml:lang="en-US" xlink:label="dei_EntityInteractiveDataCurrent_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Entity Interactive Data Current</link:label>
    <link:loc xlink:href="https://xbrl.sec.gov/dei/2020/dei-2020-01-31.xsd#dei_CurrentFiscalYearEndDate" xlink:type="locator" xlink:label="dei_CurrentFiscalYearEndDate" />
    <link:label xml:lang="en-US" xlink:label="dei_CurrentFiscalYearEndDate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Current Fiscal Year End Date</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="dei_CurrentFiscalYearEndDate" xlink:to="dei_CurrentFiscalYearEndDate_lbl" />
    <link:label xml:lang="en-US" xlink:label="dei_CurrentFiscalYearEndDate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Current Fiscal Year End Date</link:label>
    <link:loc xlink:href="https://xbrl.sec.gov/dei/2020/dei-2020-01-31.xsd#dei_EntityFilerCategory" xlink:type="locator" xlink:label="dei_EntityFilerCategory" />
    <link:label xml:lang="en-US" xlink:label="dei_EntityFilerCategory_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Entity Filer Category</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="dei_EntityFilerCategory" xlink:to="dei_EntityFilerCategory_lbl" />
    <link:label xml:lang="en-US" xlink:label="dei_EntityFilerCategory_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Entity Filer Category</link:label>
    <link:loc xlink:href="https://xbrl.sec.gov/dei/2020/dei-2020-01-31.xsd#dei_EntityWellKnownSeasonedIssuer" xlink:type="locator" xlink:label="dei_EntityWellKnownSeasonedIssuer" />
    <link:label xml:lang="en-US" xlink:label="dei_EntityWellKnownSeasonedIssuer_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Entity Well Known Seasoned Issuer</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="dei_EntityWellKnownSeasonedIssuer" xlink:to="dei_EntityWellKnownSeasonedIssuer_lbl" />
    <link:label xml:lang="en-US" xlink:label="dei_EntityWellKnownSeasonedIssuer_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Entity Well Known Seasoned Issuer</link:label>
    <link:loc xlink:href="https://xbrl.sec.gov/dei/2020/dei-2020-01-31.xsd#dei_EntityPublicFloat" xlink:type="locator" xlink:label="dei_EntityPublicFloat" />
    <link:label xml:lang="en-US" xlink:label="dei_EntityPublicFloat_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Entity Public Float</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="dei_EntityPublicFloat" xlink:to="dei_EntityPublicFloat_lbl" />
    <link:label xml:lang="en-US" xlink:label="dei_EntityPublicFloat_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Entity Public Float</link:label>
    <link:loc xlink:href="https://xbrl.sec.gov/dei/2020/dei-2020-01-31.xsd#dei_EntityCommonStockSharesOutstanding" xlink:type="locator" xlink:label="dei_EntityCommonStockSharesOutstanding" />
    <link:label xml:lang="en-US" xlink:label="dei_EntityCommonStockSharesOutstanding_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Entity Common Stock Shares Outstanding</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="dei_EntityCommonStockSharesOutstanding" xlink:to="dei_EntityCommonStockSharesOutstanding_lbl" />
    <link:label xml:lang="en-US" xlink:label="dei_EntityCommonStockSharesOutstanding_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Entity Common Stock Shares Outstanding</link:label>
    <link:loc xlink:href="https://xbrl.sec.gov/dei/2020/dei-2020-01-31.xsd#dei_EntityShellCompany" xlink:type="locator" xlink:label="dei_EntityShellCompany" />
    <link:label xml:lang="en-US" xlink:label="dei_EntityShellCompany_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Entity Shell Company</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="dei_EntityShellCompany" xlink:to="dei_EntityShellCompany_lbl" />
    <link:label xml:lang="en-US" xlink:label="dei_EntityShellCompany_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Entity Shell Company</link:label>
    <link:loc xlink:href="https://xbrl.sec.gov/dei/2020/dei-2020-01-31.xsd#dei_EntitySmallBusiness" xlink:type="locator" xlink:label="dei_EntitySmallBusiness" />
    <link:label xml:lang="en-US" xlink:label="dei_EntitySmallBusiness_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Entity Small Business</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="dei_EntitySmallBusiness" xlink:to="dei_EntitySmallBusiness_lbl" />
    <link:label xml:lang="en-US" xlink:label="dei_EntitySmallBusiness_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Entity Small Business</link:label>
    <link:loc xlink:href="https://xbrl.sec.gov/dei/2020/dei-2020-01-31.xsd#dei_EntityEmergingGrowthCompany" xlink:type="locator" xlink:label="dei_EntityEmergingGrowthCompany" />
    <link:label xml:lang="en-US" xlink:label="dei_EntityEmergingGrowthCompany_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Entity Emerging Growth Company</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="dei_EntityEmergingGrowthCompany" xlink:to="dei_EntityEmergingGrowthCompany_lbl" />
    <link:label xml:lang="en-US" xlink:label="dei_EntityEmergingGrowthCompany_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Entity Emerging Growth Company</link:label>
    <link:loc xlink:href="https://xbrl.sec.gov/dei/2020/dei-2020-01-31.xsd#dei_EntityFileNumber" xlink:type="locator" xlink:label="dei_EntityFileNumber" />
    <link:label xml:lang="en-US" xlink:label="dei_EntityFileNumber_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Entity File Number</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="dei_EntityFileNumber" xlink:to="dei_EntityFileNumber_lbl" />
    <link:label xml:lang="en-US" xlink:label="dei_EntityFileNumber_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Entity File Number</link:label>
    <link:loc xlink:href="https://xbrl.sec.gov/dei/2020/dei-2020-01-31.xsd#dei_EntityIncorporationStateCountryCode" xlink:type="locator" xlink:label="dei_EntityIncorporationStateCountryCode" />
    <link:label xml:lang="en-US" xlink:label="dei_EntityIncorporationStateCountryCode_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Entity Incorporation, State or Country Code</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="dei_EntityIncorporationStateCountryCode" xlink:to="dei_EntityIncorporationStateCountryCode_lbl" />
    <link:label xml:lang="en-US" xlink:label="dei_EntityIncorporationStateCountryCode_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Entity Incorporation State Country Code</link:label>
    <link:loc xlink:href="https://xbrl.sec.gov/dei/2020/dei-2020-01-31.xsd#dei_EntityTaxIdentificationNumber" xlink:type="locator" xlink:label="dei_EntityTaxIdentificationNumber" />
    <link:label xml:lang="en-US" xlink:label="dei_EntityTaxIdentificationNumber_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Entity Tax Identification Number</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="dei_EntityTaxIdentificationNumber" xlink:to="dei_EntityTaxIdentificationNumber_lbl" />
    <link:label xml:lang="en-US" xlink:label="dei_EntityTaxIdentificationNumber_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Entity Tax Identification Number</link:label>
    <link:loc xlink:href="https://xbrl.sec.gov/dei/2020/dei-2020-01-31.xsd#dei_EntityAddressAddressLine1" xlink:type="locator" xlink:label="dei_EntityAddressAddressLine1" />
    <link:label xml:lang="en-US" xlink:label="dei_EntityAddressAddressLine1_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Entity Address, Address Line One</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="dei_EntityAddressAddressLine1" xlink:to="dei_EntityAddressAddressLine1_lbl" />
    <link:label xml:lang="en-US" xlink:label="dei_EntityAddressAddressLine1_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Entity Address Address Line1</link:label>
    <link:loc xlink:href="https://xbrl.sec.gov/dei/2020/dei-2020-01-31.xsd#dei_EntityAddressCityOrTown" xlink:type="locator" xlink:label="dei_EntityAddressCityOrTown" />
    <link:label xml:lang="en-US" xlink:label="dei_EntityAddressCityOrTown_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Entity Address, City or Town</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="dei_EntityAddressCityOrTown" xlink:to="dei_EntityAddressCityOrTown_lbl" />
    <link:label xml:lang="en-US" xlink:label="dei_EntityAddressCityOrTown_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Entity Address City Or Town</link:label>
    <link:loc xlink:href="https://xbrl.sec.gov/dei/2020/dei-2020-01-31.xsd#dei_EntityAddressStateOrProvince" xlink:type="locator" xlink:label="dei_EntityAddressStateOrProvince" />
    <link:label xml:lang="en-US" xlink:label="dei_EntityAddressStateOrProvince_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Entity Address, State or Province</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="dei_EntityAddressStateOrProvince" xlink:to="dei_EntityAddressStateOrProvince_lbl" />
    <link:label xml:lang="en-US" xlink:label="dei_EntityAddressStateOrProvince_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Entity Address State Or Province</link:label>
    <link:loc xlink:href="https://xbrl.sec.gov/dei/2020/dei-2020-01-31.xsd#dei_EntityAddressPostalZipCode" xlink:type="locator" xlink:label="dei_EntityAddressPostalZipCode" />
    <link:label xml:lang="en-US" xlink:label="dei_EntityAddressPostalZipCode_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Entity Address, Postal Zip Code</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="dei_EntityAddressPostalZipCode" xlink:to="dei_EntityAddressPostalZipCode_lbl" />
    <link:label xml:lang="en-US" xlink:label="dei_EntityAddressPostalZipCode_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Entity Address Postal Zip Code</link:label>
    <link:loc xlink:href="https://xbrl.sec.gov/dei/2020/dei-2020-01-31.xsd#dei_CityAreaCode" xlink:type="locator" xlink:label="dei_CityAreaCode" />
    <link:label xml:lang="en-US" xlink:label="dei_CityAreaCode_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">City Area Code</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="dei_CityAreaCode" xlink:to="dei_CityAreaCode_lbl" />
    <link:label xml:lang="en-US" xlink:label="dei_CityAreaCode_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">City Area Code</link:label>
    <link:loc xlink:href="https://xbrl.sec.gov/dei/2020/dei-2020-01-31.xsd#dei_LocalPhoneNumber" xlink:type="locator" xlink:label="dei_LocalPhoneNumber" />
    <link:label xml:lang="en-US" xlink:label="dei_LocalPhoneNumber_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Local Phone Number</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="dei_LocalPhoneNumber" xlink:to="dei_LocalPhoneNumber_lbl" />
    <link:label xml:lang="en-US" xlink:label="dei_LocalPhoneNumber_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Local Phone Number</link:label>
    <link:loc xlink:href="https://xbrl.sec.gov/dei/2020/dei-2020-01-31.xsd#dei_DocumentAnnualReport" xlink:type="locator" xlink:label="dei_DocumentAnnualReport" />
    <link:label xml:lang="en-US" xlink:label="dei_DocumentAnnualReport_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Document Annual Report</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="dei_DocumentAnnualReport" xlink:to="dei_DocumentAnnualReport_lbl" />
    <link:label xml:lang="en-US" xlink:label="dei_DocumentAnnualReport_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Document Annual Report</link:label>
    <link:loc xlink:href="https://xbrl.sec.gov/dei/2020/dei-2020-01-31.xsd#dei_DocumentTransitionReport" xlink:type="locator" xlink:label="dei_DocumentTransitionReport" />
    <link:label xml:lang="en-US" xlink:label="dei_DocumentTransitionReport_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Document Transition Report</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="dei_DocumentTransitionReport" xlink:to="dei_DocumentTransitionReport_lbl" />
    <link:label xml:lang="en-US" xlink:label="dei_DocumentTransitionReport_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Document Transition Report</link:label>
    <link:loc xlink:href="https://xbrl.sec.gov/dei/2020/dei-2020-01-31.xsd#dei_IcfrAuditorAttestationFlag" xlink:type="locator" xlink:label="dei_IcfrAuditorAttestationFlag" />
    <link:label xml:lang="en-US" xlink:label="dei_IcfrAuditorAttestationFlag_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">ICFR Auditor Attestation Flag</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="dei_IcfrAuditorAttestationFlag" xlink:to="dei_IcfrAuditorAttestationFlag_lbl" />
    <link:label xml:lang="en-US" xlink:label="dei_IcfrAuditorAttestationFlag_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Icfr Auditor Attestation Flag</link:label>
    <link:loc xlink:href="https://xbrl.sec.gov/dei/2020/dei-2020-01-31.xsd#dei_Security12bTitle" xlink:type="locator" xlink:label="dei_Security12bTitle" />
    <link:label xml:lang="en-US" xlink:label="dei_Security12bTitle_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Title of each class</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="dei_Security12bTitle" xlink:to="dei_Security12bTitle_lbl" />
    <link:label xml:lang="en-US" xlink:label="dei_Security12bTitle_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Security12b Title</link:label>
    <link:loc xlink:href="https://xbrl.sec.gov/dei/2020/dei-2020-01-31.xsd#dei_TradingSymbol" xlink:type="locator" xlink:label="dei_TradingSymbol" />
    <link:label xml:lang="en-US" xlink:label="dei_TradingSymbol_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Trading Symbol</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="dei_TradingSymbol" xlink:to="dei_TradingSymbol_lbl" />
    <link:label xml:lang="en-US" xlink:label="dei_TradingSymbol_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Trading Symbol</link:label>
    <link:loc xlink:href="https://xbrl.sec.gov/dei/2020/dei-2020-01-31.xsd#dei_SecurityExchangeName" xlink:type="locator" xlink:label="dei_SecurityExchangeName" />
    <link:label xml:lang="en-US" xlink:label="dei_SecurityExchangeName_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Name of each exchange on which registered</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="dei_SecurityExchangeName" xlink:to="dei_SecurityExchangeName_lbl" />
    <link:label xml:lang="en-US" xlink:label="dei_SecurityExchangeName_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Security Exchange Name</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_StatementOfFinancialPositionAbstract" xlink:type="locator" xlink:label="us-gaap_StatementOfFinancialPositionAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_StatementOfFinancialPositionAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Statement Of Financial Position [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_StatementOfFinancialPositionAbstract" xlink:to="us-gaap_StatementOfFinancialPositionAbstract_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AssetsAbstract" xlink:type="locator" xlink:label="us-gaap_AssetsAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AssetsAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">ASSETS:</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AssetsAbstract" xlink:to="us-gaap_AssetsAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AssetsAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Assets [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_CashAndDueFromBanks" xlink:type="locator" xlink:label="us-gaap_CashAndDueFromBanks" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CashAndDueFromBanks_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Cash and due from banks (including restricted balances of $1,600 and $0, respectively)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_CashAndDueFromBanks" xlink:to="us-gaap_CashAndDueFromBanks_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CashAndDueFromBanks_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Cash And Due From Banks</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_InterestBearingDepositsInBanks" xlink:type="locator" xlink:label="us-gaap_InterestBearingDepositsInBanks" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestBearingDepositsInBanks_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Interest-earning deposits (including restricted balances of $0 and $0, respectively)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_InterestBearingDepositsInBanks" xlink:to="us-gaap_InterestBearingDepositsInBanks_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestBearingDepositsInBanks_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Interest Bearing Deposits In Banks</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalents" xlink:type="locator" xlink:label="us-gaap_CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalents" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalents_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Total cash and cash equivalents</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalents" xlink:to="us-gaap_CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalents_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalents_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Cash Cash Equivalents Restricted Cash And Restricted Cash Equivalents</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AvailableForSaleDebtAndEquitySecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Available-for-sale securities, at fair value</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AvailableForSaleDebtAndEquitySecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Available For Sale Debt And Equity Securities</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_HeldToMaturitySecurities" xlink:type="locator" xlink:label="us-gaap_HeldToMaturitySecurities" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Held-to-maturity securities, at amortized cost (fair value of $174,935 and $124,148 , respectively)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_HeldToMaturitySecurities" xlink:to="us-gaap_HeldToMaturitySecurities_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Held To Maturity Securities</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_MarketableSecurities" xlink:type="locator" xlink:label="us-gaap_MarketableSecurities" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_MarketableSecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Marketable equity securities, at fair value</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_MarketableSecurities" xlink:to="us-gaap_MarketableSecurities_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_MarketableSecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Marketable Securities</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FederalHomeLoanBankStock" xlink:type="locator" xlink:label="us-gaap_FederalHomeLoanBankStock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FederalHomeLoanBankStock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Federal Home Loan Bank stock, at cost</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FederalHomeLoanBankStock" xlink:to="us-gaap_FederalHomeLoanBankStock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FederalHomeLoanBankStock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Federal Home Loan Bank Stock</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LoansAndLeasesReceivableNetOfDeferredIncome" xlink:type="locator" xlink:label="us-gaap_LoansAndLeasesReceivableNetOfDeferredIncome" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LoansAndLeasesReceivableNetOfDeferredIncome_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Loans</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LoansAndLeasesReceivableNetOfDeferredIncome" xlink:to="us-gaap_LoansAndLeasesReceivableNetOfDeferredIncome_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LoansAndLeasesReceivableNetOfDeferredIncome_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Loans And Leases Receivable Net Of Deferred Income</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LoansReceivableHeldForSaleAmount" xlink:type="locator" xlink:label="us-gaap_LoansReceivableHeldForSaleAmount" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LoansReceivableHeldForSaleAmount_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Loans held-for-sale</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LoansReceivableHeldForSaleAmount" xlink:to="us-gaap_LoansReceivableHeldForSaleAmount_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LoansReceivableHeldForSaleAmount_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Loans Receivable Held For Sale Amount</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LoansAndLeasesReceivableAllowance" xlink:type="locator" xlink:label="us-gaap_LoansAndLeasesReceivableAllowance" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LoansAndLeasesReceivableAllowance_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Less: Allowance for loan losses</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LoansAndLeasesReceivableAllowance" xlink:to="us-gaap_LoansAndLeasesReceivableAllowance_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LoansAndLeasesReceivableAllowance_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Loans And Leases Receivable Allowance</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_NotesReceivableNet" xlink:type="locator" xlink:label="us-gaap_NotesReceivableNet" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_NotesReceivableNet_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Loans receivable, net</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_NotesReceivableNet" xlink:to="us-gaap_NotesReceivableNet_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_NotesReceivableNet_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Notes Receivable Net</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_PropertyPlantAndEquipmentNet" xlink:type="locator" xlink:label="us-gaap_PropertyPlantAndEquipmentNet" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PropertyPlantAndEquipmentNet_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Premises and equipment, net</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_PropertyPlantAndEquipmentNet" xlink:to="us-gaap_PropertyPlantAndEquipmentNet_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PropertyPlantAndEquipmentNet_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Property Plant And Equipment Net</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_OperatingLeaseRightOfUseAsset" xlink:type="locator" xlink:label="us-gaap_OperatingLeaseRightOfUseAsset" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OperatingLeaseRightOfUseAsset_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Operating lease right-of-use assets</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_OperatingLeaseRightOfUseAsset" xlink:to="us-gaap_OperatingLeaseRightOfUseAsset_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OperatingLeaseRightOfUseAsset_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Operating Lease Right Of Use Asset</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_InterestReceivableCurrent" xlink:type="locator" xlink:label="us-gaap_InterestReceivableCurrent" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestReceivableCurrent_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Accrued interest receivable</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_InterestReceivableCurrent" xlink:to="us-gaap_InterestReceivableCurrent_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestReceivableCurrent_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Interest Receivable Current</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ForeclosedAssets" xlink:type="locator" xlink:label="us-gaap_ForeclosedAssets" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ForeclosedAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Foreclosed real estate</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ForeclosedAssets" xlink:to="us-gaap_ForeclosedAssets_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ForeclosedAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Foreclosed Assets</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_IntangibleAssetsNetExcludingGoodwill" xlink:type="locator" xlink:label="us-gaap_IntangibleAssetsNetExcludingGoodwill" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_IntangibleAssetsNetExcludingGoodwill_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Intangible assets, net</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_IntangibleAssetsNetExcludingGoodwill" xlink:to="us-gaap_IntangibleAssetsNetExcludingGoodwill_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_IntangibleAssetsNetExcludingGoodwill_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Intangible Assets Net Excluding Goodwill</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_Goodwill" xlink:type="locator" xlink:label="us-gaap_Goodwill" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_Goodwill_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Goodwill</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_Goodwill" xlink:to="us-gaap_Goodwill_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_Goodwill_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Goodwill</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_BankOwnedLifeInsurance" xlink:type="locator" xlink:label="us-gaap_BankOwnedLifeInsurance" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_BankOwnedLifeInsurance_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Bank owned life insurance</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_BankOwnedLifeInsurance" xlink:to="us-gaap_BankOwnedLifeInsurance_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_BankOwnedLifeInsurance_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Bank Owned Life Insurance</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_OtherAssets" xlink:type="locator" xlink:label="us-gaap_OtherAssets" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Other assets</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_OtherAssets" xlink:to="us-gaap_OtherAssets_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Other Assets</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_Assets" xlink:type="locator" xlink:label="us-gaap_Assets" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_Assets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Total assets</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_Assets" xlink:to="us-gaap_Assets_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_Assets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Assets</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LiabilitiesAndStockholdersEquityAbstract" xlink:type="locator" xlink:label="us-gaap_LiabilitiesAndStockholdersEquityAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LiabilitiesAndStockholdersEquityAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">LIABILITIES AND SHAREHOLDERS&apos; EQUITY:</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LiabilitiesAndStockholdersEquityAbstract" xlink:to="us-gaap_LiabilitiesAndStockholdersEquityAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LiabilitiesAndStockholdersEquityAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Liabilities And Stockholders Equity [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DepositsAbstract" xlink:type="locator" xlink:label="us-gaap_DepositsAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DepositsAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Deposits:</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DepositsAbstract" xlink:to="us-gaap_DepositsAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DepositsAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deposits [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_InterestBearingDepositLiabilities" xlink:type="locator" xlink:label="us-gaap_InterestBearingDepositLiabilities" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestBearingDepositLiabilities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Interest-bearing</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_InterestBearingDepositLiabilities" xlink:to="us-gaap_InterestBearingDepositLiabilities_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestBearingDepositLiabilities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Interest Bearing Deposit Liabilities</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_NoninterestBearingDepositLiabilities" xlink:type="locator" xlink:label="us-gaap_NoninterestBearingDepositLiabilities" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_NoninterestBearingDepositLiabilities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Noninterest-bearing</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_NoninterestBearingDepositLiabilities" xlink:to="us-gaap_NoninterestBearingDepositLiabilities_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_NoninterestBearingDepositLiabilities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Noninterest Bearing Deposit Liabilities</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_Deposits" xlink:type="locator" xlink:label="us-gaap_Deposits" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_Deposits_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Total deposits</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_Deposits" xlink:to="us-gaap_Deposits_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_Deposits_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deposits</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ShortTermBorrowings" xlink:type="locator" xlink:label="us-gaap_ShortTermBorrowings" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShortTermBorrowings_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Short-term borrowings</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ShortTermBorrowings" xlink:to="us-gaap_ShortTermBorrowings_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShortTermBorrowings_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Short Term Borrowings</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LongTermDebtNoncurrent" xlink:type="locator" xlink:label="us-gaap_LongTermDebtNoncurrent" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LongTermDebtNoncurrent_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Long-term borrowings</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LongTermDebtNoncurrent" xlink:to="us-gaap_LongTermDebtNoncurrent_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LongTermDebtNoncurrent_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Long Term Debt Noncurrent</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_JuniorSubordinatedDebentureOwedToUnconsolidatedSubsidiaryTrust" xlink:type="locator" xlink:label="us-gaap_JuniorSubordinatedDebentureOwedToUnconsolidatedSubsidiaryTrust" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_JuniorSubordinatedDebentureOwedToUnconsolidatedSubsidiaryTrust_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Subordinated loans</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_JuniorSubordinatedDebentureOwedToUnconsolidatedSubsidiaryTrust" xlink:to="us-gaap_JuniorSubordinatedDebentureOwedToUnconsolidatedSubsidiaryTrust_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_JuniorSubordinatedDebentureOwedToUnconsolidatedSubsidiaryTrust_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Junior Subordinated Debenture Owed To Unconsolidated Subsidiary Trust</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_InterestPayableCurrentAndNoncurrent" xlink:type="locator" xlink:label="us-gaap_InterestPayableCurrentAndNoncurrent" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestPayableCurrentAndNoncurrent_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Accrued interest payable</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_InterestPayableCurrentAndNoncurrent" xlink:to="us-gaap_InterestPayableCurrentAndNoncurrent_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestPayableCurrentAndNoncurrent_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Interest Payable Current And Noncurrent</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_OperatingLeaseLiability" xlink:type="locator" xlink:label="us-gaap_OperatingLeaseLiability" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OperatingLeaseLiability_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Operating lease liabilities</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_OperatingLeaseLiability" xlink:to="us-gaap_OperatingLeaseLiability_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OperatingLeaseLiability_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Operating Lease Liability</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_OtherLiabilities" xlink:type="locator" xlink:label="us-gaap_OtherLiabilities" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherLiabilities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Other liabilities</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_OtherLiabilities" xlink:to="us-gaap_OtherLiabilities_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherLiabilities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Other Liabilities</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_Liabilities" xlink:type="locator" xlink:label="us-gaap_Liabilities" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_Liabilities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Total liabilities</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_Liabilities" xlink:to="us-gaap_Liabilities_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_Liabilities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Liabilities</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_StockholdersEquityAbstract" xlink:type="locator" xlink:label="us-gaap_StockholdersEquityAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_StockholdersEquityAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Shareholders&apos; equity:</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_StockholdersEquityAbstract" xlink:to="us-gaap_StockholdersEquityAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_StockholdersEquityAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Stockholders Equity [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_PreferredStockValue" xlink:type="locator" xlink:label="us-gaap_PreferredStockValue" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PreferredStockValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Preferred stock, par value $0.01 per share; no liquidation preference; 10,000,000 and 10,000,000 shares authorized, respectively; 1,380,283 and 1,155,283 shares issued and outstanding, respectively</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_PreferredStockValue" xlink:to="us-gaap_PreferredStockValue_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PreferredStockValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Preferred Stock Value</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_CommonStockValue" xlink:type="locator" xlink:label="us-gaap_CommonStockValue" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CommonStockValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Common stock, par value $0.01; 25,000,000 authorized shares; 4,531,383 and 4,709,238 shares issued and outstanding, respectively</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_CommonStockValue" xlink:to="us-gaap_CommonStockValue_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CommonStockValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Common Stock Value</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AdditionalPaidInCapital" xlink:type="locator" xlink:label="us-gaap_AdditionalPaidInCapital" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AdditionalPaidInCapital_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Additional paid in capital</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AdditionalPaidInCapital" xlink:to="us-gaap_AdditionalPaidInCapital_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AdditionalPaidInCapital_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Additional Paid In Capital</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_RetainedEarningsAccumulatedDeficit" xlink:type="locator" xlink:label="us-gaap_RetainedEarningsAccumulatedDeficit" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RetainedEarningsAccumulatedDeficit_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Retained earnings</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_RetainedEarningsAccumulatedDeficit" xlink:to="us-gaap_RetainedEarningsAccumulatedDeficit_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RetainedEarningsAccumulatedDeficit_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Retained Earnings Accumulated Deficit</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AccumulatedOtherComprehensiveIncomeLossNetOfTax" xlink:type="locator" xlink:label="us-gaap_AccumulatedOtherComprehensiveIncomeLossNetOfTax" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AccumulatedOtherComprehensiveIncomeLossNetOfTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Accumulated other comprehensive loss</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AccumulatedOtherComprehensiveIncomeLossNetOfTax" xlink:to="us-gaap_AccumulatedOtherComprehensiveIncomeLossNetOfTax_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AccumulatedOtherComprehensiveIncomeLossNetOfTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Accumulated Other Comprehensive Income Loss Net Of Tax</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_UnearnedESOPShares" xlink:type="locator" xlink:label="us-gaap_UnearnedESOPShares" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_UnearnedESOPShares_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Unearned ESOP</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_UnearnedESOPShares" xlink:to="us-gaap_UnearnedESOPShares_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_UnearnedESOPShares_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Unearned E S O P Shares</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_StockholdersEquity" xlink:type="locator" xlink:label="us-gaap_StockholdersEquity" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_StockholdersEquity_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Total Pathfinder Bancorp, Inc. shareholders&apos; equity</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_StockholdersEquity" xlink:to="us-gaap_StockholdersEquity_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_StockholdersEquity_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Stockholders Equity</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_MinorityInterest" xlink:type="locator" xlink:label="us-gaap_MinorityInterest" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_MinorityInterest_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Noncontrolling interest</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_MinorityInterest" xlink:to="us-gaap_MinorityInterest_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_MinorityInterest_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Minority Interest</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_StockholdersEquityIncludingPortionAttributableToNoncontrollingInterest" xlink:type="locator" xlink:label="us-gaap_StockholdersEquityIncludingPortionAttributableToNoncontrollingInterest" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_StockholdersEquityIncludingPortionAttributableToNoncontrollingInterest_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Total equity</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_StockholdersEquityIncludingPortionAttributableToNoncontrollingInterest" xlink:to="us-gaap_StockholdersEquityIncludingPortionAttributableToNoncontrollingInterest_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_StockholdersEquityIncludingPortionAttributableToNoncontrollingInterest_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Stockholders Equity Including Portion Attributable To Noncontrolling Interest</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LiabilitiesAndStockholdersEquity" xlink:type="locator" xlink:label="us-gaap_LiabilitiesAndStockholdersEquity" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LiabilitiesAndStockholdersEquity_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Total liabilities and shareholders&apos; equity</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LiabilitiesAndStockholdersEquity" xlink:to="us-gaap_LiabilitiesAndStockholdersEquity_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LiabilitiesAndStockholdersEquity_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Liabilities And Stockholders Equity</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_RestrictedCashAndCashEquivalents" xlink:type="locator" xlink:label="us-gaap_RestrictedCashAndCashEquivalents" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RestrictedCashAndCashEquivalents_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Cash and due from banks including restricted cash</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_RestrictedCashAndCashEquivalents" xlink:to="us-gaap_RestrictedCashAndCashEquivalents_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RestrictedCashAndCashEquivalents_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Restricted Cash And Cash Equivalents</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_RestrictedCash" xlink:type="locator" xlink:label="us-gaap_RestrictedCash" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RestrictedCash_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Restricted Cash</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_RestrictedCash" xlink:to="us-gaap_RestrictedCash_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RestrictedCash_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Restricted Cash</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_HeldToMaturitySecuritiesFairValue" xlink:type="locator" xlink:label="us-gaap_HeldToMaturitySecuritiesFairValue" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesFairValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Held-to-maturity securities at fair value</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_HeldToMaturitySecuritiesFairValue" xlink:to="us-gaap_HeldToMaturitySecuritiesFairValue_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesFairValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Held To Maturity Securities Fair Value</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_PreferredStockParOrStatedValuePerShare" xlink:type="locator" xlink:label="us-gaap_PreferredStockParOrStatedValuePerShare" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PreferredStockParOrStatedValuePerShare_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Preferred stock, par value (in dollars per share)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_PreferredStockParOrStatedValuePerShare" xlink:to="us-gaap_PreferredStockParOrStatedValuePerShare_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PreferredStockParOrStatedValuePerShare_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Preferred Stock Par Or Stated Value Per Share</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_PreferredStockLiquidationPreferenceValue" xlink:type="locator" xlink:label="us-gaap_PreferredStockLiquidationPreferenceValue" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PreferredStockLiquidationPreferenceValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Preferred stock, liquidation preference</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_PreferredStockLiquidationPreferenceValue" xlink:to="us-gaap_PreferredStockLiquidationPreferenceValue_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PreferredStockLiquidationPreferenceValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Preferred Stock Liquidation Preference Value</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_PreferredStockSharesAuthorized" xlink:type="locator" xlink:label="us-gaap_PreferredStockSharesAuthorized" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PreferredStockSharesAuthorized_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Preferred stock, shares authorized (in shares)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_PreferredStockSharesAuthorized" xlink:to="us-gaap_PreferredStockSharesAuthorized_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PreferredStockSharesAuthorized_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Preferred Stock Shares Authorized</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_PreferredStockSharesIssued" xlink:type="locator" xlink:label="us-gaap_PreferredStockSharesIssued" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PreferredStockSharesIssued_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Preferred stock, shares issued (in shares)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_PreferredStockSharesIssued" xlink:to="us-gaap_PreferredStockSharesIssued_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PreferredStockSharesIssued_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Preferred Stock Shares Issued</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_PreferredStockSharesOutstanding" xlink:type="locator" xlink:label="us-gaap_PreferredStockSharesOutstanding" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PreferredStockSharesOutstanding_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Preferred stock, shares outstanding (in shares)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_PreferredStockSharesOutstanding" xlink:to="us-gaap_PreferredStockSharesOutstanding_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PreferredStockSharesOutstanding_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Preferred Stock Shares Outstanding</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_CommonStockParOrStatedValuePerShare" xlink:type="locator" xlink:label="us-gaap_CommonStockParOrStatedValuePerShare" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CommonStockParOrStatedValuePerShare_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Common stock, par value (in dollars per share)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_CommonStockParOrStatedValuePerShare" xlink:to="us-gaap_CommonStockParOrStatedValuePerShare_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CommonStockParOrStatedValuePerShare_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Common Stock Par Or Stated Value Per Share</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_CommonStockSharesAuthorized" xlink:type="locator" xlink:label="us-gaap_CommonStockSharesAuthorized" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CommonStockSharesAuthorized_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Common stock, shares authorized (in shares)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_CommonStockSharesAuthorized" xlink:to="us-gaap_CommonStockSharesAuthorized_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CommonStockSharesAuthorized_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Common Stock Shares Authorized</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_CommonStockSharesIssued" xlink:type="locator" xlink:label="us-gaap_CommonStockSharesIssued" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CommonStockSharesIssued_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Common stock, shares issued (in shares)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_CommonStockSharesIssued" xlink:to="us-gaap_CommonStockSharesIssued_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CommonStockSharesIssued_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Common Stock Shares Issued</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_CommonStockSharesOutstanding" xlink:type="locator" xlink:label="us-gaap_CommonStockSharesOutstanding" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CommonStockSharesOutstanding_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Common stock, shares outstanding (in shares)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_CommonStockSharesOutstanding" xlink:to="us-gaap_CommonStockSharesOutstanding_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CommonStockSharesOutstanding_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Common Stock Shares Outstanding</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_CommunityServiceActivities" xlink:type="locator" xlink:label="pbhc_CommunityServiceActivities" />
    <link:label xml:lang="en-US" xlink:label="pbhc_CommunityServiceActivities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Community service activities</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_CommunityServiceActivities" xlink:to="pbhc_CommunityServiceActivities_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_IncomeStatementAbstract" xlink:type="locator" xlink:label="us-gaap_IncomeStatementAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncomeStatementAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Income Statement [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_IncomeStatementAbstract" xlink:to="us-gaap_IncomeStatementAbstract_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_StatementTable" xlink:type="locator" xlink:label="us-gaap_StatementTable" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_StatementTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Statement [Table]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_StatementTable" xlink:to="us-gaap_StatementTable_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_StatementTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Statement [Table]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/srt/2020/elts/srt-2020-01-31.xsd#srt_ProductOrServiceAxis" xlink:type="locator" xlink:label="srt_ProductOrServiceAxis" />
    <link:label xml:lang="en-US" xlink:label="srt_ProductOrServiceAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Product and Service</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="srt_ProductOrServiceAxis" xlink:to="srt_ProductOrServiceAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="srt_ProductOrServiceAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Product Or Service [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/srt/2020/elts/srt-2020-01-31.xsd#srt_ProductsAndServicesDomain" xlink:type="locator" xlink:label="srt_ProductsAndServicesDomain" />
    <link:label xml:lang="en-US" xlink:label="srt_ProductsAndServicesDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Product and Service</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="srt_ProductsAndServicesDomain" xlink:to="srt_ProductsAndServicesDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="srt_ProductsAndServicesDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Products And Services [Domain]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DepositAccountMember" xlink:type="locator" xlink:label="us-gaap_DepositAccountMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DepositAccountMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Service Charges on Deposit Accounts [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DepositAccountMember" xlink:to="us-gaap_DepositAccountMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DepositAccountMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deposit Account [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_BankServicingMember" xlink:type="locator" xlink:label="us-gaap_BankServicingMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_BankServicingMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Loan Servicing Fees [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_BankServicingMember" xlink:to="us-gaap_BankServicingMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_BankServicingMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Bank Servicing [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DebitCardMember" xlink:type="locator" xlink:label="us-gaap_DebitCardMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebitCardMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Debit Card Interchange Fees [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DebitCardMember" xlink:to="us-gaap_DebitCardMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebitCardMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Debit Card [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FinancialServiceOtherMember" xlink:type="locator" xlink:label="us-gaap_FinancialServiceOtherMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancialServiceOtherMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Other Charges, Commissions &amp; Fees [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FinancialServiceOtherMember" xlink:to="us-gaap_FinancialServiceOtherMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancialServiceOtherMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Financial Service Other [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_StatementLineItems" xlink:type="locator" xlink:label="us-gaap_StatementLineItems" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_StatementLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Statement [Line Items]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_StatementLineItems" xlink:to="us-gaap_StatementLineItems_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_StatementLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Statement [Line Items]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_InterestAndDividendIncomeOperatingAbstract" xlink:type="locator" xlink:label="us-gaap_InterestAndDividendIncomeOperatingAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestAndDividendIncomeOperatingAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Interest and dividend income:</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_InterestAndDividendIncomeOperatingAbstract" xlink:to="us-gaap_InterestAndDividendIncomeOperatingAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestAndDividendIncomeOperatingAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Interest And Dividend Income Operating [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_InterestAndFeeIncomeLoansAndLeases" xlink:type="locator" xlink:label="us-gaap_InterestAndFeeIncomeLoansAndLeases" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestAndFeeIncomeLoansAndLeases_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Loans, including fees</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_InterestAndFeeIncomeLoansAndLeases" xlink:to="us-gaap_InterestAndFeeIncomeLoansAndLeases_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestAndFeeIncomeLoansAndLeases_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Interest And Fee Income Loans And Leases</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_InterestAndDividendIncomeSecuritiesByTaxableStatusAbstract" xlink:type="locator" xlink:label="us-gaap_InterestAndDividendIncomeSecuritiesByTaxableStatusAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestAndDividendIncomeSecuritiesByTaxableStatusAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Debt securities:</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_InterestAndDividendIncomeSecuritiesByTaxableStatusAbstract" xlink:to="us-gaap_InterestAndDividendIncomeSecuritiesByTaxableStatusAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestAndDividendIncomeSecuritiesByTaxableStatusAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Interest And Dividend Income Securities By Taxable Status [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_InterestIncomeSecuritiesTaxable" xlink:type="locator" xlink:label="us-gaap_InterestIncomeSecuritiesTaxable" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestIncomeSecuritiesTaxable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Taxable</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_InterestIncomeSecuritiesTaxable" xlink:to="us-gaap_InterestIncomeSecuritiesTaxable_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestIncomeSecuritiesTaxable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Interest Income Securities Taxable</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_InterestIncomeSecuritiesTaxExempt" xlink:type="locator" xlink:label="us-gaap_InterestIncomeSecuritiesTaxExempt" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestIncomeSecuritiesTaxExempt_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Tax-exempt</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_InterestIncomeSecuritiesTaxExempt" xlink:to="us-gaap_InterestIncomeSecuritiesTaxExempt_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestIncomeSecuritiesTaxExempt_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Interest Income Securities Tax Exempt</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DividendIncomeOperating" xlink:type="locator" xlink:label="us-gaap_DividendIncomeOperating" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DividendIncomeOperating_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Dividends</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DividendIncomeOperating" xlink:to="us-gaap_DividendIncomeOperating_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DividendIncomeOperating_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Dividend Income Operating</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_InterestIncomeFederalFundsSold" xlink:type="locator" xlink:label="us-gaap_InterestIncomeFederalFundsSold" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestIncomeFederalFundsSold_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Federal funds sold and interest earning deposits</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_InterestIncomeFederalFundsSold" xlink:to="us-gaap_InterestIncomeFederalFundsSold_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestIncomeFederalFundsSold_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Interest Income Federal Funds Sold</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_InterestAndDividendIncomeOperating" xlink:type="locator" xlink:label="us-gaap_InterestAndDividendIncomeOperating" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestAndDividendIncomeOperating_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Total interest and dividend income</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_InterestAndDividendIncomeOperating" xlink:to="us-gaap_InterestAndDividendIncomeOperating_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestAndDividendIncomeOperating_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Interest And Dividend Income Operating</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_InterestExpenseAbstract" xlink:type="locator" xlink:label="us-gaap_InterestExpenseAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestExpenseAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Interest expense:</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_InterestExpenseAbstract" xlink:to="us-gaap_InterestExpenseAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestExpenseAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Interest Expense [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_InterestExpenseDeposits" xlink:type="locator" xlink:label="us-gaap_InterestExpenseDeposits" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestExpenseDeposits_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Interest on deposits</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_InterestExpenseDeposits" xlink:to="us-gaap_InterestExpenseDeposits_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestExpenseDeposits_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Interest Expense Deposits</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_InterestExpenseShortTermBorrowings" xlink:type="locator" xlink:label="us-gaap_InterestExpenseShortTermBorrowings" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestExpenseShortTermBorrowings_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Interest on short-term borrowings</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_InterestExpenseShortTermBorrowings" xlink:to="us-gaap_InterestExpenseShortTermBorrowings_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestExpenseShortTermBorrowings_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Interest Expense Short Term Borrowings</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_InterestExpenseLongTermDebt" xlink:type="locator" xlink:label="us-gaap_InterestExpenseLongTermDebt" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestExpenseLongTermDebt_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Interest on long-term borrowings</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_InterestExpenseLongTermDebt" xlink:to="us-gaap_InterestExpenseLongTermDebt_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestExpenseLongTermDebt_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Interest Expense Long Term Debt</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_InterestExpenseDebt" xlink:type="locator" xlink:label="us-gaap_InterestExpenseDebt" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestExpenseDebt_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Interest on subordinated loans</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_InterestExpenseDebt" xlink:to="us-gaap_InterestExpenseDebt_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestExpenseDebt_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Interest Expense Debt</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_InterestExpense" xlink:type="locator" xlink:label="us-gaap_InterestExpense" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestExpense_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Total interest expense</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_InterestExpense" xlink:to="us-gaap_InterestExpense_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestExpense_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Interest Expense</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_InterestIncomeExpenseNet" xlink:type="locator" xlink:label="us-gaap_InterestIncomeExpenseNet" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestIncomeExpenseNet_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Net interest income</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_InterestIncomeExpenseNet" xlink:to="us-gaap_InterestIncomeExpenseNet_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestIncomeExpenseNet_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Interest Income Expense Net</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ProvisionForLoanAndLeaseLosses" xlink:type="locator" xlink:label="us-gaap_ProvisionForLoanAndLeaseLosses" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ProvisionForLoanAndLeaseLosses_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Provision for loan losses</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ProvisionForLoanAndLeaseLosses" xlink:to="us-gaap_ProvisionForLoanAndLeaseLosses_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ProvisionForLoanAndLeaseLosses_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Provision For Loan And Lease Losses</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_InterestIncomeExpenseAfterProvisionForLoanLoss" xlink:type="locator" xlink:label="us-gaap_InterestIncomeExpenseAfterProvisionForLoanLoss" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestIncomeExpenseAfterProvisionForLoanLoss_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Net interest income after provision for loan losses</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_InterestIncomeExpenseAfterProvisionForLoanLoss" xlink:to="us-gaap_InterestIncomeExpenseAfterProvisionForLoanLoss_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestIncomeExpenseAfterProvisionForLoanLoss_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Interest Income Expense After Provision For Loan Loss</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_NoninterestIncomeAbstract" xlink:type="locator" xlink:label="us-gaap_NoninterestIncomeAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_NoninterestIncomeAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Noninterest income:</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_NoninterestIncomeAbstract" xlink:to="us-gaap_NoninterestIncomeAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_NoninterestIncomeAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Noninterest Income [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_RevenueFromContractWithCustomerIncludingAssessedTax" xlink:type="locator" xlink:label="us-gaap_RevenueFromContractWithCustomerIncludingAssessedTax" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RevenueFromContractWithCustomerIncludingAssessedTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Noninterest income</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_RevenueFromContractWithCustomerIncludingAssessedTax" xlink:to="us-gaap_RevenueFromContractWithCustomerIncludingAssessedTax_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RevenueFromContractWithCustomerIncludingAssessedTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Revenue From Contract With Customer Including Assessed Tax</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_BankOwnedLifeInsuranceIncome" xlink:type="locator" xlink:label="us-gaap_BankOwnedLifeInsuranceIncome" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_BankOwnedLifeInsuranceIncome_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Earnings and gain on bank owned life insurance</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_BankOwnedLifeInsuranceIncome" xlink:to="us-gaap_BankOwnedLifeInsuranceIncome_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_BankOwnedLifeInsuranceIncome_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Bank Owned Life Insurance Income</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_GainLossOnSaleOfInvestments" xlink:type="locator" xlink:label="us-gaap_GainLossOnSaleOfInvestments" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_GainLossOnSaleOfInvestments_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Net gains on sales and redemptions of investment securities</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_GainLossOnSaleOfInvestments" xlink:to="us-gaap_GainLossOnSaleOfInvestments_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_GainLossOnSaleOfInvestments_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Gain Loss On Sale Of Investments</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_EquitySecuritiesFvNiGainLoss" xlink:type="locator" xlink:label="us-gaap_EquitySecuritiesFvNiGainLoss" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EquitySecuritiesFvNiGainLoss_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">(Losses) gains on marketable equity securities</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_EquitySecuritiesFvNiGainLoss" xlink:to="us-gaap_EquitySecuritiesFvNiGainLoss_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EquitySecuritiesFvNiGainLoss_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Equity Securities Fv Ni Gain Loss</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NetGainsLossesOnSalesOfLoansAndForeclosedRealEstate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Net gains on sales of loans and foreclosed real estate</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NetGainsLossesOnSalesOfLoansAndForeclosedRealEstate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Net Gains Losses On Sales Of Loans And Foreclosed Real Estate</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_InsuranceServicesRevenue" xlink:type="locator" xlink:label="us-gaap_InsuranceServicesRevenue" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InsuranceServicesRevenue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Insurance agency revenue</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_InsuranceServicesRevenue" xlink:to="us-gaap_InsuranceServicesRevenue_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InsuranceServicesRevenue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Insurance Services Revenue</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_NoninterestIncome" xlink:type="locator" xlink:label="us-gaap_NoninterestIncome" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_NoninterestIncome_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Total noninterest income</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_NoninterestIncome" xlink:to="us-gaap_NoninterestIncome_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_NoninterestIncome_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Noninterest Income</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_NoninterestExpenseAbstract" xlink:type="locator" xlink:label="us-gaap_NoninterestExpenseAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_NoninterestExpenseAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Noninterest expense:</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_NoninterestExpenseAbstract" xlink:to="us-gaap_NoninterestExpenseAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_NoninterestExpenseAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Noninterest Expense [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_SalariesAndWages" xlink:type="locator" xlink:label="us-gaap_SalariesAndWages" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SalariesAndWages_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Salaries and employee benefits</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_SalariesAndWages" xlink:to="us-gaap_SalariesAndWages_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SalariesAndWages_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Salaries And Wages</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_OccupancyNet" xlink:type="locator" xlink:label="us-gaap_OccupancyNet" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OccupancyNet_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Building and occupancy</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_OccupancyNet" xlink:to="us-gaap_OccupancyNet_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OccupancyNet_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Occupancy Net</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_InformationTechnologyAndDataProcessing" xlink:type="locator" xlink:label="us-gaap_InformationTechnologyAndDataProcessing" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InformationTechnologyAndDataProcessing_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Data processing</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_InformationTechnologyAndDataProcessing" xlink:to="us-gaap_InformationTechnologyAndDataProcessing_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InformationTechnologyAndDataProcessing_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Information Technology And Data Processing</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ProfessionalFees" xlink:type="locator" xlink:label="us-gaap_ProfessionalFees" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ProfessionalFees_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Professional and other services</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ProfessionalFees" xlink:to="us-gaap_ProfessionalFees_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ProfessionalFees_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Professional Fees</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AdvertisingExpense" xlink:type="locator" xlink:label="us-gaap_AdvertisingExpense" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AdvertisingExpense_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Advertising</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AdvertisingExpense" xlink:to="us-gaap_AdvertisingExpense_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AdvertisingExpense_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Advertising Expense</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FederalDepositInsuranceCorporationPremiumExpense" xlink:type="locator" xlink:label="us-gaap_FederalDepositInsuranceCorporationPremiumExpense" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FederalDepositInsuranceCorporationPremiumExpense_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">FDIC assessments</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FederalDepositInsuranceCorporationPremiumExpense" xlink:to="us-gaap_FederalDepositInsuranceCorporationPremiumExpense_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FederalDepositInsuranceCorporationPremiumExpense_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Federal Deposit Insurance Corporation Premium Expense</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_GeneralAndAdministrativeExpense" xlink:type="locator" xlink:label="us-gaap_GeneralAndAdministrativeExpense" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_GeneralAndAdministrativeExpense_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Audits and exams</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_GeneralAndAdministrativeExpense" xlink:to="us-gaap_GeneralAndAdministrativeExpense_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_GeneralAndAdministrativeExpense_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">General And Administrative Expense</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_GeneralInsuranceExpense" xlink:type="locator" xlink:label="us-gaap_GeneralInsuranceExpense" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_GeneralInsuranceExpense_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Insurance agency expense</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_GeneralInsuranceExpense" xlink:to="us-gaap_GeneralInsuranceExpense_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_GeneralInsuranceExpense_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">General Insurance Expense</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_CommunityServiceActivities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Community service activities</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_CommunityServiceActivities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Community Service Activities</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ForeclosedRealEstateExpense" xlink:type="locator" xlink:label="us-gaap_ForeclosedRealEstateExpense" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ForeclosedRealEstateExpense_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Foreclosed real estate expenses</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ForeclosedRealEstateExpense" xlink:to="us-gaap_ForeclosedRealEstateExpense_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ForeclosedRealEstateExpense_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Foreclosed Real Estate Expense</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_OtherNoninterestExpense" xlink:type="locator" xlink:label="us-gaap_OtherNoninterestExpense" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherNoninterestExpense_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Other expenses</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_OtherNoninterestExpense" xlink:to="us-gaap_OtherNoninterestExpense_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherNoninterestExpense_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Other Noninterest Expense</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_NoninterestExpense" xlink:type="locator" xlink:label="us-gaap_NoninterestExpense" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_NoninterestExpense_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Total noninterest expenses</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_NoninterestExpense" xlink:to="us-gaap_NoninterestExpense_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_NoninterestExpense_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Noninterest Expense</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_IncomeLossFromContinuingOperationsBeforeIncomeTaxesExtraordinaryItemsNoncontrollingInterest" xlink:type="locator" xlink:label="us-gaap_IncomeLossFromContinuingOperationsBeforeIncomeTaxesExtraordinaryItemsNoncontrollingInterest" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncomeLossFromContinuingOperationsBeforeIncomeTaxesExtraordinaryItemsNoncontrollingInterest_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Income before income taxes</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_IncomeLossFromContinuingOperationsBeforeIncomeTaxesExtraordinaryItemsNoncontrollingInterest" xlink:to="us-gaap_IncomeLossFromContinuingOperationsBeforeIncomeTaxesExtraordinaryItemsNoncontrollingInterest_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncomeLossFromContinuingOperationsBeforeIncomeTaxesExtraordinaryItemsNoncontrollingInterest_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Income Loss From Continuing Operations Before Income Taxes Extraordinary Items Noncontrolling Interest</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_IncomeTaxExpenseBenefit" xlink:type="locator" xlink:label="us-gaap_IncomeTaxExpenseBenefit" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncomeTaxExpenseBenefit_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Provision for income taxes</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_IncomeTaxExpenseBenefit" xlink:to="us-gaap_IncomeTaxExpenseBenefit_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncomeTaxExpenseBenefit_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Income Tax Expense Benefit</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ProfitLoss" xlink:type="locator" xlink:label="us-gaap_ProfitLoss" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ProfitLoss_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Net income attributable to noncontrolling interest and Pathfinder Bancorp, Inc.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ProfitLoss" xlink:to="us-gaap_ProfitLoss_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ProfitLoss_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Profit Loss</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_NetIncomeLossAttributableToNoncontrollingInterest" xlink:type="locator" xlink:label="us-gaap_NetIncomeLossAttributableToNoncontrollingInterest" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_NetIncomeLossAttributableToNoncontrollingInterest_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Net income attributable to noncontrolling interest</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_NetIncomeLossAttributableToNoncontrollingInterest" xlink:to="us-gaap_NetIncomeLossAttributableToNoncontrollingInterest_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_NetIncomeLossAttributableToNoncontrollingInterest_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Net Income Loss Attributable To Noncontrolling Interest</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_NetIncomeLoss" xlink:type="locator" xlink:label="us-gaap_NetIncomeLoss" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_NetIncomeLoss_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Net income attributable to Pathfinder Bancorp, Inc.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_NetIncomeLoss" xlink:to="us-gaap_NetIncomeLoss_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_NetIncomeLoss_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Net Income Loss</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_PreferredStockDividendsIncomeStatementImpact" xlink:type="locator" xlink:label="us-gaap_PreferredStockDividendsIncomeStatementImpact" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PreferredStockDividendsIncomeStatementImpact_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Convertible preferred stock dividends</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_PreferredStockDividendsIncomeStatementImpact" xlink:to="us-gaap_PreferredStockDividendsIncomeStatementImpact_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PreferredStockDividendsIncomeStatementImpact_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Preferred Stock Dividends Income Statement Impact</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_WarrantDividendsIncomeStatementImpact_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Warrant dividends</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_WarrantDividendsIncomeStatementImpact_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Warrant Dividends Income Statement Impact</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_UndistributedEarningsLossAllocatedToParticipatingSecuritiesBasic" xlink:type="locator" xlink:label="us-gaap_UndistributedEarningsLossAllocatedToParticipatingSecuritiesBasic" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_UndistributedEarningsLossAllocatedToParticipatingSecuritiesBasic_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Undistributed earnings allocated to participating securities</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_UndistributedEarningsLossAllocatedToParticipatingSecuritiesBasic" xlink:to="us-gaap_UndistributedEarningsLossAllocatedToParticipatingSecuritiesBasic_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_UndistributedEarningsLossAllocatedToParticipatingSecuritiesBasic_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Undistributed Earnings Loss Allocated To Participating Securities Basic</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_NetIncomeLossAvailableToCommonStockholdersBasic" xlink:type="locator" xlink:label="us-gaap_NetIncomeLossAvailableToCommonStockholdersBasic" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_NetIncomeLossAvailableToCommonStockholdersBasic_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Net income available to common shareholders</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_NetIncomeLossAvailableToCommonStockholdersBasic" xlink:to="us-gaap_NetIncomeLossAvailableToCommonStockholdersBasic_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_NetIncomeLossAvailableToCommonStockholdersBasic_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Net Income Loss Available To Common Stockholders Basic</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_EarningsPerShareBasic" xlink:type="locator" xlink:label="us-gaap_EarningsPerShareBasic" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EarningsPerShareBasic_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Earnings per common share - basic</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_EarningsPerShareBasic" xlink:to="us-gaap_EarningsPerShareBasic_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EarningsPerShareBasic_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Earnings Per Share Basic</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_EarningsPerShareDiluted" xlink:type="locator" xlink:label="us-gaap_EarningsPerShareDiluted" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EarningsPerShareDiluted_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Earnings per common share - diluted</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_EarningsPerShareDiluted" xlink:to="us-gaap_EarningsPerShareDiluted_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EarningsPerShareDiluted_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Earnings Per Share Diluted</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_CommonStockDividendsPerShareDeclared" xlink:type="locator" xlink:label="us-gaap_CommonStockDividendsPerShareDeclared" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CommonStockDividendsPerShareDeclared_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Dividends per common share</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_CommonStockDividendsPerShareDeclared" xlink:to="us-gaap_CommonStockDividendsPerShareDeclared_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CommonStockDividendsPerShareDeclared_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Common Stock Dividends Per Share Declared</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_UnrealizedHoldingLossesArisingDuringThePeriod" xlink:type="locator" xlink:label="pbhc_UnrealizedHoldingLossesArisingDuringThePeriod" />
    <link:label xml:lang="en-US" xlink:label="pbhc_UnrealizedHoldingLossesArisingDuringThePeriod_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Unrealized holding losses arising during the period.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_UnrealizedHoldingLossesArisingDuringThePeriod" xlink:to="pbhc_UnrealizedHoldingLossesArisingDuringThePeriod_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_OtherComprehensiveIncomeLossUnrealizedHoldingGainLossOnSecuritiesTransferredToHeldToMaturityArisingDuringPeriodBeforeTax" xlink:type="locator" xlink:label="pbhc_OtherComprehensiveIncomeLossUnrealizedHoldingGainLossOnSecuritiesTransferredToHeldToMaturityArisingDuringPeriodBeforeTax" />
    <link:label xml:lang="en-US" xlink:label="pbhc_OtherComprehensiveIncomeLossUnrealizedHoldingGainLossOnSecuritiesTransferredToHeldToMaturityArisingDuringPeriodBeforeTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Before tax and reclassification adjustments amount of unrealized holding gain (loss) and transfers to held-to-maturity securities.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_OtherComprehensiveIncomeLossUnrealizedHoldingGainLossOnSecuritiesTransferredToHeldToMaturityArisingDuringPeriodBeforeTax" xlink:to="pbhc_OtherComprehensiveIncomeLossUnrealizedHoldingGainLossOnSecuritiesTransferredToHeldToMaturityArisingDuringPeriodBeforeTax_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_OtherComprehensiveIncomeLossUnrealizedHoldingGainLossOnSecuritiesTransferredToHeldToMaturityArisingDuringPeriodTax" xlink:type="locator" xlink:label="pbhc_OtherComprehensiveIncomeLossUnrealizedHoldingGainLossOnSecuritiesTransferredToHeldToMaturityArisingDuringPeriodTax" />
    <link:label xml:lang="en-US" xlink:label="pbhc_OtherComprehensiveIncomeLossUnrealizedHoldingGainLossOnSecuritiesTransferredToHeldToMaturityArisingDuringPeriodTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Tax effect of unrealized holding gain (loss) before reclassification adjustments and transfers to Held-to-maturity.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_OtherComprehensiveIncomeLossUnrealizedHoldingGainLossOnSecuritiesTransferredToHeldToMaturityArisingDuringPeriodTax" xlink:to="pbhc_OtherComprehensiveIncomeLossUnrealizedHoldingGainLossOnSecuritiesTransferredToHeldToMaturityArisingDuringPeriodTax_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_StatementOfIncomeAndComprehensiveIncomeAbstract" xlink:type="locator" xlink:label="us-gaap_StatementOfIncomeAndComprehensiveIncomeAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_StatementOfIncomeAndComprehensiveIncomeAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Consolidated Statements of Comprehensive Income [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_StatementOfIncomeAndComprehensiveIncomeAbstract" xlink:to="us-gaap_StatementOfIncomeAndComprehensiveIncomeAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_StatementOfIncomeAndComprehensiveIncomeAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Statement Of Income And Comprehensive Income [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_ProfitLoss_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Net Income</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_OtherComprehensiveIncomeLossPensionAndOtherPostretirementBenefitPlansAdjustmentBeforeReclassificationAdjustmentsAndTaxAbstract" xlink:type="locator" xlink:label="us-gaap_OtherComprehensiveIncomeLossPensionAndOtherPostretirementBenefitPlansAdjustmentBeforeReclassificationAdjustmentsAndTaxAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherComprehensiveIncomeLossPensionAndOtherPostretirementBenefitPlansAdjustmentBeforeReclassificationAdjustmentsAndTaxAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Retirement Plans:</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_OtherComprehensiveIncomeLossPensionAndOtherPostretirementBenefitPlansAdjustmentBeforeReclassificationAdjustmentsAndTaxAbstract" xlink:to="us-gaap_OtherComprehensiveIncomeLossPensionAndOtherPostretirementBenefitPlansAdjustmentBeforeReclassificationAdjustmentsAndTaxAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherComprehensiveIncomeLossPensionAndOtherPostretirementBenefitPlansAdjustmentBeforeReclassificationAdjustmentsAndTaxAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Other Comprehensive Income Loss Pension And Other Postretirement Benefit Plans Adjustment Before Reclassification Adjustments And Tax [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_OtherComprehensiveIncomeLossPensionAndOtherPostretirementBenefitPlansNetUnamortizedGainLossArisingDuringPeriodBeforeTax" xlink:type="locator" xlink:label="us-gaap_OtherComprehensiveIncomeLossPensionAndOtherPostretirementBenefitPlansNetUnamortizedGainLossArisingDuringPeriodBeforeTax" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherComprehensiveIncomeLossPensionAndOtherPostretirementBenefitPlansNetUnamortizedGainLossArisingDuringPeriodBeforeTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Retirement plan net losses recognized in plan expenses</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_OtherComprehensiveIncomeLossPensionAndOtherPostretirementBenefitPlansNetUnamortizedGainLossArisingDuringPeriodBeforeTax" xlink:to="us-gaap_OtherComprehensiveIncomeLossPensionAndOtherPostretirementBenefitPlansNetUnamortizedGainLossArisingDuringPeriodBeforeTax_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherComprehensiveIncomeLossPensionAndOtherPostretirementBenefitPlansNetUnamortizedGainLossArisingDuringPeriodBeforeTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Other Comprehensive Income Loss Pension And Other Postretirement Benefit Plans Net Unamortized Gain Loss Arising During Period Before Tax</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_OtherComprehensiveIncomeLossFinalizationOfPensionAndNonPensionPostretirementPlanValuationBeforeTax" xlink:type="locator" xlink:label="us-gaap_OtherComprehensiveIncomeLossFinalizationOfPensionAndNonPensionPostretirementPlanValuationBeforeTax" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherComprehensiveIncomeLossFinalizationOfPensionAndNonPensionPostretirementPlanValuationBeforeTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Plan gains not recognized in plan expenses</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_OtherComprehensiveIncomeLossFinalizationOfPensionAndNonPensionPostretirementPlanValuationBeforeTax" xlink:to="us-gaap_OtherComprehensiveIncomeLossFinalizationOfPensionAndNonPensionPostretirementPlanValuationBeforeTax_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherComprehensiveIncomeLossFinalizationOfPensionAndNonPensionPostretirementPlanValuationBeforeTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Other Comprehensive Income Loss Finalization Of Pension And Non Pension Postretirement Plan Valuation Before Tax</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_OtherComprehensiveIncomeLossPensionAndOtherPostretirementBenefitPlansAdjustmentBeforeTax" xlink:type="locator" xlink:label="us-gaap_OtherComprehensiveIncomeLossPensionAndOtherPostretirementBenefitPlansAdjustmentBeforeTax" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherComprehensiveIncomeLossPensionAndOtherPostretirementBenefitPlansAdjustmentBeforeTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedTotalLabel">Net unrealized gain on retirement plans</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_OtherComprehensiveIncomeLossPensionAndOtherPostretirementBenefitPlansAdjustmentBeforeTax" xlink:to="us-gaap_OtherComprehensiveIncomeLossPensionAndOtherPostretirementBenefitPlansAdjustmentBeforeTax_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherComprehensiveIncomeLossPensionAndOtherPostretirementBenefitPlansAdjustmentBeforeTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Other Comprehensive Income Loss Pension And Other Postretirement Benefit Plans Adjustment Before Tax</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_OtherComprehensiveIncomeAvailableForSaleSecuritiesAdjustmentBeforeTaxPeriodIncreaseDecreaseAbstract" xlink:type="locator" xlink:label="us-gaap_OtherComprehensiveIncomeAvailableForSaleSecuritiesAdjustmentBeforeTaxPeriodIncreaseDecreaseAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherComprehensiveIncomeAvailableForSaleSecuritiesAdjustmentBeforeTaxPeriodIncreaseDecreaseAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Unrealized holding gains on available-for-sale securities</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_OtherComprehensiveIncomeAvailableForSaleSecuritiesAdjustmentBeforeTaxPeriodIncreaseDecreaseAbstract" xlink:to="us-gaap_OtherComprehensiveIncomeAvailableForSaleSecuritiesAdjustmentBeforeTaxPeriodIncreaseDecreaseAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherComprehensiveIncomeAvailableForSaleSecuritiesAdjustmentBeforeTaxPeriodIncreaseDecreaseAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Other Comprehensive Income Available For Sale Securities Adjustment Before Tax Period Increase Decrease [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_OtherComprehensiveIncomeUnrealizedHoldingGainLossOnSecuritiesArisingDuringPeriodBeforeTax" xlink:type="locator" xlink:label="us-gaap_OtherComprehensiveIncomeUnrealizedHoldingGainLossOnSecuritiesArisingDuringPeriodBeforeTax" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherComprehensiveIncomeUnrealizedHoldingGainLossOnSecuritiesArisingDuringPeriodBeforeTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Unrealized holding gains arising during the period</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_OtherComprehensiveIncomeUnrealizedHoldingGainLossOnSecuritiesArisingDuringPeriodBeforeTax" xlink:to="us-gaap_OtherComprehensiveIncomeUnrealizedHoldingGainLossOnSecuritiesArisingDuringPeriodBeforeTax_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherComprehensiveIncomeUnrealizedHoldingGainLossOnSecuritiesArisingDuringPeriodBeforeTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Other Comprehensive Income Unrealized Holding Gain Loss On Securities Arising During Period Before Tax</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_OtherComprehensiveIncomeLossReclassificationAdjustmentFromAOCIForSaleOfSecuritiesBeforeTax" xlink:type="locator" xlink:label="us-gaap_OtherComprehensiveIncomeLossReclassificationAdjustmentFromAOCIForSaleOfSecuritiesBeforeTax" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherComprehensiveIncomeLossReclassificationAdjustmentFromAOCIForSaleOfSecuritiesBeforeTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Reclassification adjustment for net gains included in net income</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_OtherComprehensiveIncomeLossReclassificationAdjustmentFromAOCIForSaleOfSecuritiesBeforeTax" xlink:to="us-gaap_OtherComprehensiveIncomeLossReclassificationAdjustmentFromAOCIForSaleOfSecuritiesBeforeTax_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherComprehensiveIncomeLossReclassificationAdjustmentFromAOCIForSaleOfSecuritiesBeforeTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Other Comprehensive Income Loss Reclassification Adjustment From A O C I For Sale Of Securities Before Tax</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_OtherComprehensiveIncomeLossAvailableForSaleSecuritiesAdjustmentBeforeTax" xlink:type="locator" xlink:label="us-gaap_OtherComprehensiveIncomeLossAvailableForSaleSecuritiesAdjustmentBeforeTax" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherComprehensiveIncomeLossAvailableForSaleSecuritiesAdjustmentBeforeTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Net unrealized gains on available-for-sale securities</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_OtherComprehensiveIncomeLossAvailableForSaleSecuritiesAdjustmentBeforeTax" xlink:to="us-gaap_OtherComprehensiveIncomeLossAvailableForSaleSecuritiesAdjustmentBeforeTax_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherComprehensiveIncomeLossAvailableForSaleSecuritiesAdjustmentBeforeTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Other Comprehensive Income Loss Available For Sale Securities Adjustment Before Tax</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_OtherComprehensiveIncomeDerivativesQualifyingAsHedgesBeforeTaxPeriodIncreaseDecreaseAbstract" xlink:type="locator" xlink:label="us-gaap_OtherComprehensiveIncomeDerivativesQualifyingAsHedgesBeforeTaxPeriodIncreaseDecreaseAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherComprehensiveIncomeDerivativesQualifyingAsHedgesBeforeTaxPeriodIncreaseDecreaseAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Derivatives and hedging activities:</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_OtherComprehensiveIncomeDerivativesQualifyingAsHedgesBeforeTaxPeriodIncreaseDecreaseAbstract" xlink:to="us-gaap_OtherComprehensiveIncomeDerivativesQualifyingAsHedgesBeforeTaxPeriodIncreaseDecreaseAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherComprehensiveIncomeDerivativesQualifyingAsHedgesBeforeTaxPeriodIncreaseDecreaseAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Other Comprehensive Income Derivatives Qualifying As Hedges Before Tax Period Increase Decrease [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_UnrealizedHoldingLossesArisingDuringThePeriod_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Unrealized holding losses arising during the period</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_UnrealizedHoldingLossesArisingDuringThePeriod_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Unrealized Holding Losses Arising During The Period</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_OtherComprehensiveIncomeLossCashFlowHedgeGainLossAfterReclassificationBeforeTax" xlink:type="locator" xlink:label="us-gaap_OtherComprehensiveIncomeLossCashFlowHedgeGainLossAfterReclassificationBeforeTax" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherComprehensiveIncomeLossCashFlowHedgeGainLossAfterReclassificationBeforeTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Net unrealized losses on derivatives and hedging activities</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_OtherComprehensiveIncomeLossCashFlowHedgeGainLossAfterReclassificationBeforeTax" xlink:to="us-gaap_OtherComprehensiveIncomeLossCashFlowHedgeGainLossAfterReclassificationBeforeTax_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherComprehensiveIncomeLossCashFlowHedgeGainLossAfterReclassificationBeforeTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Other Comprehensive Income Loss Cash Flow Hedge Gain Loss After Reclassification Before Tax</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_OtherComprehensiveIncomeLossUnrealizedHoldingGainLossOnSecuritiesTransferredToHeldToMaturityArisingDuringPeriodBeforeTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Accretion of net unrealized loss on securities transferred to held-to-maturity</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_OtherComprehensiveIncomeLossUnrealizedHoldingGainLossOnSecuritiesTransferredToHeldToMaturityArisingDuringPeriodBeforeTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Other Comprehensive Income Loss Unrealized Holding Gain Loss On Securities Transferred To Held To Maturity Arising During Period Before Tax</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_OtherComprehensiveIncomeLossBeforeTax" xlink:type="locator" xlink:label="us-gaap_OtherComprehensiveIncomeLossBeforeTax" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherComprehensiveIncomeLossBeforeTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Other comprehensive income, before tax</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_OtherComprehensiveIncomeLossBeforeTax" xlink:to="us-gaap_OtherComprehensiveIncomeLossBeforeTax_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherComprehensiveIncomeLossBeforeTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Other Comprehensive Income Loss Before Tax</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_OtherComprehensiveIncomeLossTax" xlink:type="locator" xlink:label="us-gaap_OtherComprehensiveIncomeLossTax" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherComprehensiveIncomeLossTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Tax effect</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_OtherComprehensiveIncomeLossTax" xlink:to="us-gaap_OtherComprehensiveIncomeLossTax_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherComprehensiveIncomeLossTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Other Comprehensive Income Loss Tax</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_OtherComprehensiveIncomeLossNetOfTax" xlink:type="locator" xlink:label="us-gaap_OtherComprehensiveIncomeLossNetOfTax" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherComprehensiveIncomeLossNetOfTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Other comprehensive income, net of tax</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_OtherComprehensiveIncomeLossNetOfTax" xlink:to="us-gaap_OtherComprehensiveIncomeLossNetOfTax_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherComprehensiveIncomeLossNetOfTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Other Comprehensive Income Loss Net Of Tax</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ComprehensiveIncomeNetOfTaxIncludingPortionAttributableToNoncontrollingInterest" xlink:type="locator" xlink:label="us-gaap_ComprehensiveIncomeNetOfTaxIncludingPortionAttributableToNoncontrollingInterest" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ComprehensiveIncomeNetOfTaxIncludingPortionAttributableToNoncontrollingInterest_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Comprehensive income</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ComprehensiveIncomeNetOfTaxIncludingPortionAttributableToNoncontrollingInterest" xlink:to="us-gaap_ComprehensiveIncomeNetOfTaxIncludingPortionAttributableToNoncontrollingInterest_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ComprehensiveIncomeNetOfTaxIncludingPortionAttributableToNoncontrollingInterest_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Comprehensive Income Net Of Tax Including Portion Attributable To Noncontrolling Interest</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ComprehensiveIncomeNetOfTaxAttributableToNoncontrollingInterest" xlink:type="locator" xlink:label="us-gaap_ComprehensiveIncomeNetOfTaxAttributableToNoncontrollingInterest" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ComprehensiveIncomeNetOfTaxAttributableToNoncontrollingInterest_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Comprehensive income, attributable to noncontrolling interest</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ComprehensiveIncomeNetOfTaxAttributableToNoncontrollingInterest" xlink:to="us-gaap_ComprehensiveIncomeNetOfTaxAttributableToNoncontrollingInterest_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ComprehensiveIncomeNetOfTaxAttributableToNoncontrollingInterest_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Comprehensive Income Net Of Tax Attributable To Noncontrolling Interest</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ComprehensiveIncomeNetOfTax" xlink:type="locator" xlink:label="us-gaap_ComprehensiveIncomeNetOfTax" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ComprehensiveIncomeNetOfTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Comprehensive income attributable to Pathfinder Bancorp, Inc.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ComprehensiveIncomeNetOfTax" xlink:to="us-gaap_ComprehensiveIncomeNetOfTax_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ComprehensiveIncomeNetOfTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Comprehensive Income Net Of Tax</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_OtherComprehensiveIncomeLossTaxPortionAttributableToParentAbstract" xlink:type="locator" xlink:label="us-gaap_OtherComprehensiveIncomeLossTaxPortionAttributableToParentAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherComprehensiveIncomeLossTaxPortionAttributableToParentAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Tax Effect Allocated to Each Component of Other Comprehensive Income</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_OtherComprehensiveIncomeLossTaxPortionAttributableToParentAbstract" xlink:to="us-gaap_OtherComprehensiveIncomeLossTaxPortionAttributableToParentAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherComprehensiveIncomeLossTaxPortionAttributableToParentAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Other Comprehensive Income Loss Tax Portion Attributable To Parent [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_OtherComprehensiveIncomeDefinedBenefitPlansNetUnamortizedGainLossArisingDuringPeriodTax" xlink:type="locator" xlink:label="us-gaap_OtherComprehensiveIncomeDefinedBenefitPlansNetUnamortizedGainLossArisingDuringPeriodTax" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherComprehensiveIncomeDefinedBenefitPlansNetUnamortizedGainLossArisingDuringPeriodTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Retirement plan net losses recognized in plan expenses</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_OtherComprehensiveIncomeDefinedBenefitPlansNetUnamortizedGainLossArisingDuringPeriodTax" xlink:to="us-gaap_OtherComprehensiveIncomeDefinedBenefitPlansNetUnamortizedGainLossArisingDuringPeriodTax_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherComprehensiveIncomeDefinedBenefitPlansNetUnamortizedGainLossArisingDuringPeriodTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Other Comprehensive Income Defined Benefit Plans Net Unamortized Gain Loss Arising During Period Tax</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_OtherComprehensiveIncomeFinalizationOfPensionAndNonPensionPostretirementPlanValuationTax" xlink:type="locator" xlink:label="us-gaap_OtherComprehensiveIncomeFinalizationOfPensionAndNonPensionPostretirementPlanValuationTax" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherComprehensiveIncomeFinalizationOfPensionAndNonPensionPostretirementPlanValuationTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Plan gains not recognized in plan expenses</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_OtherComprehensiveIncomeFinalizationOfPensionAndNonPensionPostretirementPlanValuationTax" xlink:to="us-gaap_OtherComprehensiveIncomeFinalizationOfPensionAndNonPensionPostretirementPlanValuationTax_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherComprehensiveIncomeFinalizationOfPensionAndNonPensionPostretirementPlanValuationTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Other Comprehensive Income Finalization Of Pension And Non Pension Postretirement Plan Valuation Tax</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_OtherComprehensiveIncomeUnrealizedHoldingGainLossOnSecuritiesArisingDuringPeriodTax" xlink:type="locator" xlink:label="us-gaap_OtherComprehensiveIncomeUnrealizedHoldingGainLossOnSecuritiesArisingDuringPeriodTax" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherComprehensiveIncomeUnrealizedHoldingGainLossOnSecuritiesArisingDuringPeriodTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Unrealized holding gains arising during the period</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_OtherComprehensiveIncomeUnrealizedHoldingGainLossOnSecuritiesArisingDuringPeriodTax" xlink:to="us-gaap_OtherComprehensiveIncomeUnrealizedHoldingGainLossOnSecuritiesArisingDuringPeriodTax_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherComprehensiveIncomeUnrealizedHoldingGainLossOnSecuritiesArisingDuringPeriodTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Other Comprehensive Income Unrealized Holding Gain Loss On Securities Arising During Period Tax</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_OtherComprehensiveIncomeLossReclassificationAdjustmentFromAOCIForSaleOfSecuritiesTax" xlink:type="locator" xlink:label="us-gaap_OtherComprehensiveIncomeLossReclassificationAdjustmentFromAOCIForSaleOfSecuritiesTax" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherComprehensiveIncomeLossReclassificationAdjustmentFromAOCIForSaleOfSecuritiesTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Reclassification adjustment for net gains included in net income</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_OtherComprehensiveIncomeLossReclassificationAdjustmentFromAOCIForSaleOfSecuritiesTax" xlink:to="us-gaap_OtherComprehensiveIncomeLossReclassificationAdjustmentFromAOCIForSaleOfSecuritiesTax_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherComprehensiveIncomeLossReclassificationAdjustmentFromAOCIForSaleOfSecuritiesTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Other Comprehensive Income Loss Reclassification Adjustment From A O C I For Sale Of Securities Tax</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_OtherComprehensiveIncomeLossCashFlowHedgeGainLossBeforeReclassificationTax" xlink:type="locator" xlink:label="us-gaap_OtherComprehensiveIncomeLossCashFlowHedgeGainLossBeforeReclassificationTax" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherComprehensiveIncomeLossCashFlowHedgeGainLossBeforeReclassificationTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Unrealized losses on derivatives and hedging arising during the period</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_OtherComprehensiveIncomeLossCashFlowHedgeGainLossBeforeReclassificationTax" xlink:to="us-gaap_OtherComprehensiveIncomeLossCashFlowHedgeGainLossBeforeReclassificationTax_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherComprehensiveIncomeLossCashFlowHedgeGainLossBeforeReclassificationTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Other Comprehensive Income Loss Cash Flow Hedge Gain Loss Before Reclassification Tax</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_OtherComprehensiveIncomeLossUnrealizedHoldingGainLossOnSecuritiesTransferredToHeldToMaturityArisingDuringPeriodTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Accretion of net unrealized loss on securities transferred to held-to-maturity</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_OtherComprehensiveIncomeLossUnrealizedHoldingGainLossOnSecuritiesTransferredToHeldToMaturityArisingDuringPeriodTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Other Comprehensive Income Loss Unrealized Holding Gain Loss On Securities Transferred To Held To Maturity Arising During Period Tax</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherComprehensiveIncomeLossTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedTotalLabel">Income tax effect related to other comprehensive income</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ExchangeOfCommonSharesToPreferredShares" xlink:type="locator" xlink:label="pbhc_ExchangeOfCommonSharesToPreferredShares" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ExchangeOfCommonSharesToPreferredShares_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Exchange of common shares to preferred shares.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ExchangeOfCommonSharesToPreferredShares" xlink:to="pbhc_ExchangeOfCommonSharesToPreferredShares_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_EsopSharesEarned" xlink:type="locator" xlink:label="pbhc_EsopSharesEarned" />
    <link:label xml:lang="en-US" xlink:label="pbhc_EsopSharesEarned_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Employee stock ownership plan shares earned during the period.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_EsopSharesEarned" xlink:to="pbhc_EsopSharesEarned_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_DividendsWarrant" xlink:type="locator" xlink:label="pbhc_DividendsWarrant" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DividendsWarrant_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Dividends Warrant.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_DividendsWarrant" xlink:to="pbhc_DividendsWarrant_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_CapitalTransferFromAffiliates" xlink:type="locator" xlink:label="pbhc_CapitalTransferFromAffiliates" />
    <link:label xml:lang="en-US" xlink:label="pbhc_CapitalTransferFromAffiliates_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Capital transfer from affiliates.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_CapitalTransferFromAffiliates" xlink:to="pbhc_CapitalTransferFromAffiliates_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_CumulativeEffectOfAffiliateCapitalAllocation" xlink:type="locator" xlink:label="pbhc_CumulativeEffectOfAffiliateCapitalAllocation" />
    <link:label xml:lang="en-US" xlink:label="pbhc_CumulativeEffectOfAffiliateCapitalAllocation_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Cumulative effect of affiliate capital allocation.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_CumulativeEffectOfAffiliateCapitalAllocation" xlink:to="pbhc_CumulativeEffectOfAffiliateCapitalAllocation_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_DistributionsFromAffiliates" xlink:type="locator" xlink:label="pbhc_DistributionsFromAffiliates" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DistributionsFromAffiliates_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Refers to distributions from affiliates.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_DistributionsFromAffiliates" xlink:to="pbhc_DistributionsFromAffiliates_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_CommonStockIssuedDuringPeriodValuePrivatePlacementNetOfExpenses" xlink:type="locator" xlink:label="pbhc_CommonStockIssuedDuringPeriodValuePrivatePlacementNetOfExpenses" />
    <link:label xml:lang="en-US" xlink:label="pbhc_CommonStockIssuedDuringPeriodValuePrivatePlacementNetOfExpenses_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Common stock issued during period value, private placement, net of expenses.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_CommonStockIssuedDuringPeriodValuePrivatePlacementNetOfExpenses" xlink:to="pbhc_CommonStockIssuedDuringPeriodValuePrivatePlacementNetOfExpenses_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_PreferredStockIssuedDuringPeriodValuePrivatePlacementNetOfExpenses" xlink:type="locator" xlink:label="pbhc_PreferredStockIssuedDuringPeriodValuePrivatePlacementNetOfExpenses" />
    <link:label xml:lang="en-US" xlink:label="pbhc_PreferredStockIssuedDuringPeriodValuePrivatePlacementNetOfExpenses_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Preferred stock issued during period value, private placement, net of expenses.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_PreferredStockIssuedDuringPeriodValuePrivatePlacementNetOfExpenses" xlink:to="pbhc_PreferredStockIssuedDuringPeriodValuePrivatePlacementNetOfExpenses_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_StatementOfStockholdersEquityAbstract" xlink:type="locator" xlink:label="us-gaap_StatementOfStockholdersEquityAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_StatementOfStockholdersEquityAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Statement Of Stockholders Equity [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_StatementOfStockholdersEquityAbstract" xlink:to="us-gaap_StatementOfStockholdersEquityAbstract_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_StatementEquityComponentsAxis" xlink:type="locator" xlink:label="us-gaap_StatementEquityComponentsAxis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_StatementEquityComponentsAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Equity Components</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_StatementEquityComponentsAxis" xlink:to="us-gaap_StatementEquityComponentsAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_StatementEquityComponentsAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Statement Equity Components [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_EquityComponentDomain" xlink:type="locator" xlink:label="us-gaap_EquityComponentDomain" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EquityComponentDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Equity Component</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_EquityComponentDomain" xlink:to="us-gaap_EquityComponentDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EquityComponentDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Equity Component [Domain]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_PreferredStockMember" xlink:type="locator" xlink:label="us-gaap_PreferredStockMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PreferredStockMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Preferred Stock [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_PreferredStockMember" xlink:to="us-gaap_PreferredStockMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PreferredStockMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Preferred Stock [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_CommonStockMember" xlink:type="locator" xlink:label="us-gaap_CommonStockMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CommonStockMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Common Stock [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_CommonStockMember" xlink:to="us-gaap_CommonStockMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CommonStockMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Common Stock [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AdditionalPaidInCapitalMember" xlink:type="locator" xlink:label="us-gaap_AdditionalPaidInCapitalMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AdditionalPaidInCapitalMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Additional Paid in Capital [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AdditionalPaidInCapitalMember" xlink:to="us-gaap_AdditionalPaidInCapitalMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AdditionalPaidInCapitalMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Additional Paid In Capital [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_RetainedEarningsMember" xlink:type="locator" xlink:label="us-gaap_RetainedEarningsMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RetainedEarningsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Retained Earnings [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_RetainedEarningsMember" xlink:to="us-gaap_RetainedEarningsMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RetainedEarningsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Retained Earnings [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AccumulatedOtherComprehensiveIncomeMember" xlink:type="locator" xlink:label="us-gaap_AccumulatedOtherComprehensiveIncomeMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AccumulatedOtherComprehensiveIncomeMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Accumulated Other Comprehensive Loss [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AccumulatedOtherComprehensiveIncomeMember" xlink:to="us-gaap_AccumulatedOtherComprehensiveIncomeMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AccumulatedOtherComprehensiveIncomeMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Accumulated Other Comprehensive Income [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_UnearnedEsopMember" xlink:type="locator" xlink:label="pbhc_UnearnedEsopMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_UnearnedEsopMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Equity impact from Unearned ESOP.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_UnearnedEsopMember" xlink:to="pbhc_UnearnedEsopMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_UnearnedEsopMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Unearned ESOP [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_UnearnedEsopMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Unearned Esop [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_NoncontrollingInterestMember" xlink:type="locator" xlink:label="us-gaap_NoncontrollingInterestMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_NoncontrollingInterestMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Non-controlling Interest [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_NoncontrollingInterestMember" xlink:to="us-gaap_NoncontrollingInterestMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_NoncontrollingInterestMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Noncontrolling Interest [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/srt/2020/elts/srt-2020-01-31.xsd#srt_RestatementAxis" xlink:type="locator" xlink:label="srt_RestatementAxis" />
    <link:label xml:lang="en-US" xlink:label="srt_RestatementAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Revision of Prior Period</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="srt_RestatementAxis" xlink:to="srt_RestatementAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="srt_RestatementAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Restatement [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/srt/2020/elts/srt-2020-01-31.xsd#srt_RestatementDomain" xlink:type="locator" xlink:label="srt_RestatementDomain" />
    <link:label xml:lang="en-US" xlink:label="srt_RestatementDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Revision of Prior Period</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="srt_RestatementDomain" xlink:to="srt_RestatementDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="srt_RestatementDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Restatement [Domain]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/srt/2020/elts/srt-2020-01-31.xsd#srt_RevisionOfPriorPeriodAccountingStandardsUpdateAdjustmentMember" xlink:type="locator" xlink:label="srt_RevisionOfPriorPeriodAccountingStandardsUpdateAdjustmentMember" />
    <link:label xml:lang="en-US" xlink:label="srt_RevisionOfPriorPeriodAccountingStandardsUpdateAdjustmentMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Revision Of Prior Period Accounting Standards Update Adjustment [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="srt_RevisionOfPriorPeriodAccountingStandardsUpdateAdjustmentMember" xlink:to="srt_RevisionOfPriorPeriodAccountingStandardsUpdateAdjustmentMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="srt_RevisionOfPriorPeriodAccountingStandardsUpdateAdjustmentMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Revision Of Prior Period Accounting Standards Update Adjustment [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AdjustmentsForNewAccountingPronouncementsAxis" xlink:type="locator" xlink:label="us-gaap_AdjustmentsForNewAccountingPronouncementsAxis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AdjustmentsForNewAccountingPronouncementsAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Accounting Standards Update</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AdjustmentsForNewAccountingPronouncementsAxis" xlink:to="us-gaap_AdjustmentsForNewAccountingPronouncementsAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AdjustmentsForNewAccountingPronouncementsAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Adjustments For New Accounting Pronouncements [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_TypeOfAdoptionMember" xlink:type="locator" xlink:label="us-gaap_TypeOfAdoptionMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TypeOfAdoptionMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Accounting Standards Update</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_TypeOfAdoptionMember" xlink:to="us-gaap_TypeOfAdoptionMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TypeOfAdoptionMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Type Of Adoption [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AccountingStandardsUpdate201601Member" xlink:type="locator" xlink:label="us-gaap_AccountingStandardsUpdate201601Member" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AccountingStandardsUpdate201601Member_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">ASU 2016-01 [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AccountingStandardsUpdate201601Member" xlink:to="us-gaap_AccountingStandardsUpdate201601Member_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AccountingStandardsUpdate201601Member_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Accounting Standards Update201601 [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AccountingStandardsUpdate201602Member" xlink:type="locator" xlink:label="us-gaap_AccountingStandardsUpdate201602Member" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AccountingStandardsUpdate201602Member_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">ASU 2016-02 [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AccountingStandardsUpdate201602Member" xlink:to="us-gaap_AccountingStandardsUpdate201602Member_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AccountingStandardsUpdate201602Member_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Accounting Standards Update201602 [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_StockholdersEquityIncludingPortionAttributableToNoncontrollingInterest_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/periodStartLabel">Balance</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherComprehensiveIncomeLossNetOfTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Other comprehensive income (loss), net of tax</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ExchangeOfCommonSharesToPreferredShares_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Exchange of common shares to preferred shares, at par value</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ExchangeOfCommonSharesToPreferredShares_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Exchange Of Common Shares To Preferred Shares</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_EsopSharesEarned_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">ESOP shares earned</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_EsopSharesEarned_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Esop Shares Earned</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_StockIssuedDuringPeriodValueRestrictedStockAwardGross" xlink:type="locator" xlink:label="us-gaap_StockIssuedDuringPeriodValueRestrictedStockAwardGross" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_StockIssuedDuringPeriodValueRestrictedStockAwardGross_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Restricted stock units</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_StockIssuedDuringPeriodValueRestrictedStockAwardGross" xlink:to="us-gaap_StockIssuedDuringPeriodValueRestrictedStockAwardGross_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_StockIssuedDuringPeriodValueRestrictedStockAwardGross_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Stock Issued During Period Value Restricted Stock Award Gross</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AdjustmentsToAdditionalPaidInCapitalSharebasedCompensationRequisiteServicePeriodRecognitionValue" xlink:type="locator" xlink:label="us-gaap_AdjustmentsToAdditionalPaidInCapitalSharebasedCompensationRequisiteServicePeriodRecognitionValue" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AdjustmentsToAdditionalPaidInCapitalSharebasedCompensationRequisiteServicePeriodRecognitionValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Stock based compensation</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AdjustmentsToAdditionalPaidInCapitalSharebasedCompensationRequisiteServicePeriodRecognitionValue" xlink:to="us-gaap_AdjustmentsToAdditionalPaidInCapitalSharebasedCompensationRequisiteServicePeriodRecognitionValue_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AdjustmentsToAdditionalPaidInCapitalSharebasedCompensationRequisiteServicePeriodRecognitionValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Adjustments To Additional Paid In Capital Sharebased Compensation Requisite Service Period Recognition Value</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_StockIssuedDuringPeriodValueStockOptionsExercised" xlink:type="locator" xlink:label="us-gaap_StockIssuedDuringPeriodValueStockOptionsExercised" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_StockIssuedDuringPeriodValueStockOptionsExercised_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Stock options exercised</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_StockIssuedDuringPeriodValueStockOptionsExercised" xlink:to="us-gaap_StockIssuedDuringPeriodValueStockOptionsExercised_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_StockIssuedDuringPeriodValueStockOptionsExercised_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Stock Issued During Period Value Stock Options Exercised</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DividendsCommonStock" xlink:type="locator" xlink:label="us-gaap_DividendsCommonStock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DividendsCommonStock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Common stock dividends declared</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DividendsCommonStock" xlink:to="us-gaap_DividendsCommonStock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DividendsCommonStock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Dividends Common Stock</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DividendsPreferredStock" xlink:type="locator" xlink:label="us-gaap_DividendsPreferredStock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DividendsPreferredStock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Preferred stock dividends declared</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DividendsPreferredStock" xlink:to="us-gaap_DividendsPreferredStock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DividendsPreferredStock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Dividends Preferred Stock</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DividendsWarrant_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Warrant dividends declared</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DividendsWarrant_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Dividends Warrant</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_CapitalTransferFromAffiliates_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Capital transfer from affiliates</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_CapitalTransferFromAffiliates_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Capital Transfer From Affiliates</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_CumulativeEffectOfAffiliateCapitalAllocation_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Cumulative effect of affiliate capital allocation</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_CumulativeEffectOfAffiliateCapitalAllocation_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Cumulative Effect Of Affiliate Capital Allocation</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DistributionsFromAffiliates_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Distributions from affiliates</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DistributionsFromAffiliates_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Distributions From Affiliates</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_StockholdersEquityIncludingPortionAttributableToNoncontrollingInterest_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/periodEndLabel">Balance</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_CommonStockIssuedDuringPeriodValuePrivatePlacementNetOfExpenses_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Proceeds of common stock private placement, net of expenses</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_CommonStockIssuedDuringPeriodValuePrivatePlacementNetOfExpenses_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Common Stock Issued During Period Value Private Placement Net Of Expenses</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_PreferredStockIssuedDuringPeriodValuePrivatePlacementNetOfExpenses_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Proceeds of preferred stock private placement, net of expenses</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_PreferredStockIssuedDuringPeriodValuePrivatePlacementNetOfExpenses_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Preferred Stock Issued During Period Value Private Placement Net Of Expenses</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AdjustmentsToAdditionalPaidInCapitalWarrantIssued" xlink:type="locator" xlink:label="us-gaap_AdjustmentsToAdditionalPaidInCapitalWarrantIssued" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AdjustmentsToAdditionalPaidInCapitalWarrantIssued_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Effect of warrant issued from private placement</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AdjustmentsToAdditionalPaidInCapitalWarrantIssued" xlink:to="us-gaap_AdjustmentsToAdditionalPaidInCapitalWarrantIssued_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AdjustmentsToAdditionalPaidInCapitalWarrantIssued_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Adjustments To Additional Paid In Capital Warrant Issued</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_WarrantDividendsPerShareDeclared" xlink:type="locator" xlink:label="pbhc_WarrantDividendsPerShareDeclared" />
    <link:label xml:lang="en-US" xlink:label="pbhc_WarrantDividendsPerShareDeclared_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Aggregate dividends declared during the period for each share of warrant outstanding.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_WarrantDividendsPerShareDeclared" xlink:to="pbhc_WarrantDividendsPerShareDeclared_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_StatementClassOfStockAxis" xlink:type="locator" xlink:label="us-gaap_StatementClassOfStockAxis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_StatementClassOfStockAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Class of Stock</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_StatementClassOfStockAxis" xlink:to="us-gaap_StatementClassOfStockAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_StatementClassOfStockAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Statement Class Of Stock [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ClassOfStockDomain" xlink:type="locator" xlink:label="us-gaap_ClassOfStockDomain" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ClassOfStockDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Class of Stock</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ClassOfStockDomain" xlink:to="us-gaap_ClassOfStockDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ClassOfStockDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Class Of Stock [Domain]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_SeriesBPreferredStockMember" xlink:type="locator" xlink:label="us-gaap_SeriesBPreferredStockMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SeriesBPreferredStockMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Series B Preferred Stock [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_SeriesBPreferredStockMember" xlink:to="us-gaap_SeriesBPreferredStockMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SeriesBPreferredStockMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Series B Preferred Stock [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/srt/2020/elts/srt-2020-01-31.xsd#srt_OwnershipAxis" xlink:type="locator" xlink:label="srt_OwnershipAxis" />
    <link:label xml:lang="en-US" xlink:label="srt_OwnershipAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Ownership</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="srt_OwnershipAxis" xlink:to="srt_OwnershipAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="srt_OwnershipAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Ownership [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/srt/2020/elts/srt-2020-01-31.xsd#srt_OwnershipDomain" xlink:type="locator" xlink:label="srt_OwnershipDomain" />
    <link:label xml:lang="en-US" xlink:label="srt_OwnershipDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Ownership</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="srt_OwnershipDomain" xlink:to="srt_OwnershipDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="srt_OwnershipDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Ownership [Domain]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_CastleCreekMember" xlink:type="locator" xlink:label="pbhc_CastleCreekMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_CastleCreekMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Castle creek.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_CastleCreekMember" xlink:to="pbhc_CastleCreekMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_CastleCreekMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Castle Creek [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_CastleCreekMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Castle Creek [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_SubsidiarySaleOfStockAxis" xlink:type="locator" xlink:label="us-gaap_SubsidiarySaleOfStockAxis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SubsidiarySaleOfStockAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Sale of Stock</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_SubsidiarySaleOfStockAxis" xlink:to="us-gaap_SubsidiarySaleOfStockAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SubsidiarySaleOfStockAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Subsidiary Sale Of Stock [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_SaleOfStockNameOfTransactionDomain" xlink:type="locator" xlink:label="us-gaap_SaleOfStockNameOfTransactionDomain" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SaleOfStockNameOfTransactionDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Sale of Stock</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_SaleOfStockNameOfTransactionDomain" xlink:to="us-gaap_SaleOfStockNameOfTransactionDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SaleOfStockNameOfTransactionDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Sale Of Stock Name Of Transaction [Domain]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_PrivatePlacementMember" xlink:type="locator" xlink:label="us-gaap_PrivatePlacementMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PrivatePlacementMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Private Placement [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_PrivatePlacementMember" xlink:to="us-gaap_PrivatePlacementMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PrivatePlacementMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Private Placement [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_TypeOfArrangementAxis" xlink:type="locator" xlink:label="us-gaap_TypeOfArrangementAxis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TypeOfArrangementAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Collaborative Arrangement and Arrangement Other than Collaborative</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_TypeOfArrangementAxis" xlink:to="us-gaap_TypeOfArrangementAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TypeOfArrangementAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Type Of Arrangement [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ArrangementsAndNonarrangementTransactionsMember" xlink:type="locator" xlink:label="us-gaap_ArrangementsAndNonarrangementTransactionsMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ArrangementsAndNonarrangementTransactionsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Collaborative Arrangement and Arrangement Other than Collaborative</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ArrangementsAndNonarrangementTransactionsMember" xlink:to="us-gaap_ArrangementsAndNonarrangementTransactionsMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ArrangementsAndNonarrangementTransactionsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Arrangements And Nonarrangement Transactions [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_SecuritiesPurchaseAgreementMember" xlink:type="locator" xlink:label="pbhc_SecuritiesPurchaseAgreementMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_SecuritiesPurchaseAgreementMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Securities purchase agreement.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_SecuritiesPurchaseAgreementMember" xlink:to="pbhc_SecuritiesPurchaseAgreementMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_SecuritiesPurchaseAgreementMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Securities Purchase Agreement [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_SecuritiesPurchaseAgreementMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Securities Purchase Agreement [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_SubscriptionAgreementsMember" xlink:type="locator" xlink:label="pbhc_SubscriptionAgreementsMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_SubscriptionAgreementsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Subscription agreements.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_SubscriptionAgreementsMember" xlink:to="pbhc_SubscriptionAgreementsMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_SubscriptionAgreementsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Subscription Agreements [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_SubscriptionAgreementsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Subscription Agreements [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/srt/2020/elts/srt-2020-01-31.xsd#srt_TitleOfIndividualAxis" xlink:type="locator" xlink:label="srt_TitleOfIndividualAxis" />
    <link:label xml:lang="en-US" xlink:label="srt_TitleOfIndividualAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Title of Individual</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="srt_TitleOfIndividualAxis" xlink:to="srt_TitleOfIndividualAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="srt_TitleOfIndividualAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Title Of Individual [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/srt/2020/elts/srt-2020-01-31.xsd#srt_TitleOfIndividualWithRelationshipToEntityDomain" xlink:type="locator" xlink:label="srt_TitleOfIndividualWithRelationshipToEntityDomain" />
    <link:label xml:lang="en-US" xlink:label="srt_TitleOfIndividualWithRelationshipToEntityDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Title of Individual</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="srt_TitleOfIndividualWithRelationshipToEntityDomain" xlink:to="srt_TitleOfIndividualWithRelationshipToEntityDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="srt_TitleOfIndividualWithRelationshipToEntityDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Title Of Individual With Relationship To Entity [Domain]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_DirectorsAndExecutiveOfficersMember" xlink:type="locator" xlink:label="pbhc_DirectorsAndExecutiveOfficersMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DirectorsAndExecutiveOfficersMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Directors and executive officers.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_DirectorsAndExecutiveOfficersMember" xlink:to="pbhc_DirectorsAndExecutiveOfficersMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DirectorsAndExecutiveOfficersMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Directors and Executive Officers [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DirectorsAndExecutiveOfficersMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Directors And Executive Officers [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_EmployeeStockOwnershipPlanESOPSharesContributedToESOP" xlink:type="locator" xlink:label="us-gaap_EmployeeStockOwnershipPlanESOPSharesContributedToESOP" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EmployeeStockOwnershipPlanESOPSharesContributedToESOP_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">ESOP shares earned (in shares)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_EmployeeStockOwnershipPlanESOPSharesContributedToESOP" xlink:to="us-gaap_EmployeeStockOwnershipPlanESOPSharesContributedToESOP_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EmployeeStockOwnershipPlanESOPSharesContributedToESOP_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Employee Stock Ownership Plan E S O P Shares Contributed To E S O P</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_StockIssuedDuringPeriodSharesRestrictedStockAwardGross" xlink:type="locator" xlink:label="us-gaap_StockIssuedDuringPeriodSharesRestrictedStockAwardGross" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_StockIssuedDuringPeriodSharesRestrictedStockAwardGross_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Restricted stock awards/units (in shares)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_StockIssuedDuringPeriodSharesRestrictedStockAwardGross" xlink:to="us-gaap_StockIssuedDuringPeriodSharesRestrictedStockAwardGross_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_StockIssuedDuringPeriodSharesRestrictedStockAwardGross_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Stock Issued During Period Shares Restricted Stock Award Gross</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_PreferredStockDividendsPerShareDeclared" xlink:type="locator" xlink:label="us-gaap_PreferredStockDividendsPerShareDeclared" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PreferredStockDividendsPerShareDeclared_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Dividends per preferred share</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_PreferredStockDividendsPerShareDeclared" xlink:to="us-gaap_PreferredStockDividendsPerShareDeclared_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PreferredStockDividendsPerShareDeclared_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Preferred Stock Dividends Per Share Declared</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_WarrantDividendsPerShareDeclared_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Dividends per warrant</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_WarrantDividendsPerShareDeclared_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Warrant Dividends Per Share Declared</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_SaleOfStockNumberOfSharesIssuedInTransaction" xlink:type="locator" xlink:label="us-gaap_SaleOfStockNumberOfSharesIssuedInTransaction" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SaleOfStockNumberOfSharesIssuedInTransaction_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Sale of stock</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_SaleOfStockNumberOfSharesIssuedInTransaction" xlink:to="us-gaap_SaleOfStockNumberOfSharesIssuedInTransaction_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SaleOfStockNumberOfSharesIssuedInTransaction_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Sale Of Stock Number Of Shares Issued In Transaction</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ClassOfWarrantOrRightNumberOfSecuritiesCalledByEachWarrantOrRight" xlink:type="locator" xlink:label="us-gaap_ClassOfWarrantOrRightNumberOfSecuritiesCalledByEachWarrantOrRight" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ClassOfWarrantOrRightNumberOfSecuritiesCalledByEachWarrantOrRight_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Warrant to purchase common stock</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ClassOfWarrantOrRightNumberOfSecuritiesCalledByEachWarrantOrRight" xlink:to="us-gaap_ClassOfWarrantOrRightNumberOfSecuritiesCalledByEachWarrantOrRight_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ClassOfWarrantOrRightNumberOfSecuritiesCalledByEachWarrantOrRight_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Class Of Warrant Or Right Number Of Securities Called By Each Warrant Or Right</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ProceedsFromSaleOfAndRedemptionsOfAbstract" xlink:type="locator" xlink:label="pbhc_ProceedsFromSaleOfAndRedemptionsOfAbstract" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ProceedsFromSaleOfAndRedemptionsOfAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Proceeds from sales, redemptions and calls of:</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ProceedsFromSaleOfAndRedemptionsOfAbstract" xlink:to="pbhc_ProceedsFromSaleOfAndRedemptionsOfAbstract_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_StatementOfCashFlowsAbstract" xlink:type="locator" xlink:label="us-gaap_StatementOfCashFlowsAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_StatementOfCashFlowsAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Statement Of Cash Flows [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_StatementOfCashFlowsAbstract" xlink:to="us-gaap_StatementOfCashFlowsAbstract_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_NetCashProvidedByUsedInOperatingActivitiesAbstract" xlink:type="locator" xlink:label="us-gaap_NetCashProvidedByUsedInOperatingActivitiesAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_NetCashProvidedByUsedInOperatingActivitiesAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">OPERATING ACTIVITIES</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_NetCashProvidedByUsedInOperatingActivitiesAbstract" xlink:to="us-gaap_NetCashProvidedByUsedInOperatingActivitiesAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_NetCashProvidedByUsedInOperatingActivitiesAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Net Cash Provided By Used In Operating Activities [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_NetIncomeLoss_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Net income attributable to Pathfinder Bancorp, Inc.</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AdjustmentsToReconcileNetIncomeLossToCashProvidedByUsedInOperatingActivitiesAbstract" xlink:type="locator" xlink:label="us-gaap_AdjustmentsToReconcileNetIncomeLossToCashProvidedByUsedInOperatingActivitiesAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AdjustmentsToReconcileNetIncomeLossToCashProvidedByUsedInOperatingActivitiesAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Adjustments to reconcile net income to net cash flows from operating activities:</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AdjustmentsToReconcileNetIncomeLossToCashProvidedByUsedInOperatingActivitiesAbstract" xlink:to="us-gaap_AdjustmentsToReconcileNetIncomeLossToCashProvidedByUsedInOperatingActivitiesAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AdjustmentsToReconcileNetIncomeLossToCashProvidedByUsedInOperatingActivitiesAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Adjustments To Reconcile Net Income Loss To Cash Provided By Used In Operating Activities [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DeferredIncomeTaxExpenseBenefit" xlink:type="locator" xlink:label="us-gaap_DeferredIncomeTaxExpenseBenefit" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredIncomeTaxExpenseBenefit_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Deferred income (benefit) tax</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DeferredIncomeTaxExpenseBenefit" xlink:to="us-gaap_DeferredIncomeTaxExpenseBenefit_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredIncomeTaxExpenseBenefit_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deferred Income Tax Expense Benefit</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AmortizationOfOperatingLeases_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Amortization of operating leases</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AmortizationOfOperatingLeases_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Amortization Of Operating Leases</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ProceedsFromSaleOfLoansHeldForSale" xlink:type="locator" xlink:label="us-gaap_ProceedsFromSaleOfLoansHeldForSale" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ProceedsFromSaleOfLoansHeldForSale_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Proceeds from sales of loans</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ProceedsFromSaleOfLoansHeldForSale" xlink:to="us-gaap_ProceedsFromSaleOfLoansHeldForSale_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ProceedsFromSaleOfLoansHeldForSale_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Proceeds From Sale Of Loans Held For Sale</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_PaymentsForOriginationOfMortgageLoansHeldForSale" xlink:type="locator" xlink:label="us-gaap_PaymentsForOriginationOfMortgageLoansHeldForSale" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PaymentsForOriginationOfMortgageLoansHeldForSale_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Originations of loans held-for-sale</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_PaymentsForOriginationOfMortgageLoansHeldForSale" xlink:to="us-gaap_PaymentsForOriginationOfMortgageLoansHeldForSale_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PaymentsForOriginationOfMortgageLoansHeldForSale_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Payments For Origination Of Mortgage Loans Held For Sale</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_GainsLossesOnSalesOfAssetsAbstract" xlink:type="locator" xlink:label="us-gaap_GainsLossesOnSalesOfAssetsAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_GainsLossesOnSalesOfAssetsAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Realized (gains) losses on sales, redemptions and calls of:</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_GainsLossesOnSalesOfAssetsAbstract" xlink:to="us-gaap_GainsLossesOnSalesOfAssetsAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_GainsLossesOnSalesOfAssetsAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Gains Losses On Sales Of Assets [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_GainsLossesOnRealEstateAcquiredThroughForeclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Real estate acquired through foreclosure</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_GainsLossesOnRealEstateAcquiredThroughForeclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Gains Losses On Real Estate Acquired Through Foreclosure</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_GainLossOnSalesOfLoansNet" xlink:type="locator" xlink:label="us-gaap_GainLossOnSalesOfLoansNet" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_GainLossOnSalesOfLoansNet_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Loans</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_GainLossOnSalesOfLoansNet" xlink:to="us-gaap_GainLossOnSalesOfLoansNet_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_GainLossOnSalesOfLoansNet_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Gain Loss On Sales Of Loans Net</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_PaymentsForProceedsFromSaleOfAvailableForSaleSecuritiesDebt_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Available-for-sale investment securities</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_PaymentsForProceedsFromSaleOfAvailableForSaleSecuritiesDebt_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Payments For Proceeds From Sale Of Available For Sale Securities Debt</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_HeldtomaturitySecuritiesSoldSecurityRealizedGainLossExcludingOtherThanTemporaryImpairments" xlink:type="locator" xlink:label="us-gaap_HeldtomaturitySecuritiesSoldSecurityRealizedGainLossExcludingOtherThanTemporaryImpairments" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldtomaturitySecuritiesSoldSecurityRealizedGainLossExcludingOtherThanTemporaryImpairments_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Held-to-maturity investment securities</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_HeldtomaturitySecuritiesSoldSecurityRealizedGainLossExcludingOtherThanTemporaryImpairments" xlink:to="us-gaap_HeldtomaturitySecuritiesSoldSecurityRealizedGainLossExcludingOtherThanTemporaryImpairments_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldtomaturitySecuritiesSoldSecurityRealizedGainLossExcludingOtherThanTemporaryImpairments_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Heldtomaturity Securities Sold Security Realized Gain Loss Excluding Other Than Temporary Impairments</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_MarketableSecuritiesRealizedGainLoss" xlink:type="locator" xlink:label="us-gaap_MarketableSecuritiesRealizedGainLoss" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_MarketableSecuritiesRealizedGainLoss_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Marketable equity securities</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_MarketableSecuritiesRealizedGainLoss" xlink:to="us-gaap_MarketableSecuritiesRealizedGainLoss_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_MarketableSecuritiesRealizedGainLoss_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Marketable Securities Realized Gain Loss</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_Depreciation" xlink:type="locator" xlink:label="us-gaap_Depreciation" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_Depreciation_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Depreciation</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_Depreciation" xlink:to="us-gaap_Depreciation_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_Depreciation_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Depreciation</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AmortizationAccretionOfMortgageServicingRights_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Amortization of mortgage servicing rights</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AmortizationAccretionOfMortgageServicingRights_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Amortization Accretion Of Mortgage Servicing Rights</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AmortizationAccretionOfDeferredLoanCosts_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Amortization of deferred loan costs</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AmortizationAccretionOfDeferredLoanCosts_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Amortization Accretion Of Deferred Loan Costs</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AmortizationOfFinancingCostFromSubordinatedDebt_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Amortization of deferred financing from subordinated debt</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AmortizationOfFinancingCostFromSubordinatedDebt_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Amortization Of Financing Cost From Subordinated Debt</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_BankOwnedLifeInsuranceIncomeOperatingActivities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Earnings and gain on bank owned life insurance</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_BankOwnedLifeInsuranceIncomeOperatingActivities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Bank Owned Life Insurance Income Operating Activities</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AmortizationOfDebtDiscountPremium" xlink:type="locator" xlink:label="us-gaap_AmortizationOfDebtDiscountPremium" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AmortizationOfDebtDiscountPremium_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Net amortization of premiums and discounts on investment securities</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AmortizationOfDebtDiscountPremium" xlink:to="us-gaap_AmortizationOfDebtDiscountPremium_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AmortizationOfDebtDiscountPremium_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Amortization Of Debt Discount Premium</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AmortizationOfIntangibleAssets" xlink:type="locator" xlink:label="us-gaap_AmortizationOfIntangibleAssets" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AmortizationOfIntangibleAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Amortization of intangible assets</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AmortizationOfIntangibleAssets" xlink:to="us-gaap_AmortizationOfIntangibleAssets_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AmortizationOfIntangibleAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Amortization Of Intangible Assets</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_StockOptionPlanExpense" xlink:type="locator" xlink:label="us-gaap_StockOptionPlanExpense" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_StockOptionPlanExpense_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Stock based compensation and ESOP expense</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_StockOptionPlanExpense" xlink:to="us-gaap_StockOptionPlanExpense_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_StockOptionPlanExpense_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Stock Option Plan Expense</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_IncreaseDecreaseInAccruedInterestReceivableNet" xlink:type="locator" xlink:label="us-gaap_IncreaseDecreaseInAccruedInterestReceivableNet" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncreaseDecreaseInAccruedInterestReceivableNet_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Net change in accrued interest receivable</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_IncreaseDecreaseInAccruedInterestReceivableNet" xlink:to="us-gaap_IncreaseDecreaseInAccruedInterestReceivableNet_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncreaseDecreaseInAccruedInterestReceivableNet_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Increase Decrease In Accrued Interest Receivable Net</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_IncreaseDecreaseInOtherOperatingCapitalNet" xlink:type="locator" xlink:label="us-gaap_IncreaseDecreaseInOtherOperatingCapitalNet" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncreaseDecreaseInOtherOperatingCapitalNet_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Net change in other assets and liabilities</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_IncreaseDecreaseInOtherOperatingCapitalNet" xlink:to="us-gaap_IncreaseDecreaseInOtherOperatingCapitalNet_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncreaseDecreaseInOtherOperatingCapitalNet_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Increase Decrease In Other Operating Capital Net</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_NetCashProvidedByUsedInOperatingActivities" xlink:type="locator" xlink:label="us-gaap_NetCashProvidedByUsedInOperatingActivities" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_NetCashProvidedByUsedInOperatingActivities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Net cash flows from operating activities</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_NetCashProvidedByUsedInOperatingActivities" xlink:to="us-gaap_NetCashProvidedByUsedInOperatingActivities_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_NetCashProvidedByUsedInOperatingActivities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Net Cash Provided By Used In Operating Activities</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_NetCashProvidedByUsedInInvestingActivitiesAbstract" xlink:type="locator" xlink:label="us-gaap_NetCashProvidedByUsedInInvestingActivitiesAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_NetCashProvidedByUsedInInvestingActivitiesAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">INVESTING ACTIVITIES</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_NetCashProvidedByUsedInInvestingActivitiesAbstract" xlink:to="us-gaap_NetCashProvidedByUsedInInvestingActivitiesAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_NetCashProvidedByUsedInInvestingActivitiesAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Net Cash Provided By Used In Investing Activities [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_PaymentsToAcquireAvailableForSaleDebtAndEquitySecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Purchase of investment securities available-for-sale</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_PaymentsToAcquireAvailableForSaleDebtAndEquitySecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Payments To Acquire Available For Sale Debt And Equity Securities</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_PaymentsToAcquireHeldToMaturitySecurities" xlink:type="locator" xlink:label="us-gaap_PaymentsToAcquireHeldToMaturitySecurities" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PaymentsToAcquireHeldToMaturitySecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Purchase of investment securities held-to-maturity</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_PaymentsToAcquireHeldToMaturitySecurities" xlink:to="us-gaap_PaymentsToAcquireHeldToMaturitySecurities_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PaymentsToAcquireHeldToMaturitySecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Payments To Acquire Held To Maturity Securities</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_PaymentsToAcquireFederalHomeLoanBankStock" xlink:type="locator" xlink:label="us-gaap_PaymentsToAcquireFederalHomeLoanBankStock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PaymentsToAcquireFederalHomeLoanBankStock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Purchase of Federal Home Loan Bank stock</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_PaymentsToAcquireFederalHomeLoanBankStock" xlink:to="us-gaap_PaymentsToAcquireFederalHomeLoanBankStock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PaymentsToAcquireFederalHomeLoanBankStock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Payments To Acquire Federal Home Loan Bank Stock</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ProceedsFromSaleOfFederalHomeLoanBankStock" xlink:type="locator" xlink:label="us-gaap_ProceedsFromSaleOfFederalHomeLoanBankStock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ProceedsFromSaleOfFederalHomeLoanBankStock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Proceeds from redemption of Federal Home Loan Bank stock</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ProceedsFromSaleOfFederalHomeLoanBankStock" xlink:to="us-gaap_ProceedsFromSaleOfFederalHomeLoanBankStock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ProceedsFromSaleOfFederalHomeLoanBankStock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Proceeds From Sale Of Federal Home Loan Bank Stock</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ProceedsFromMaturitiesPrepaymentsAndCallsOfAvailableForSaleSecurities" xlink:type="locator" xlink:label="us-gaap_ProceedsFromMaturitiesPrepaymentsAndCallsOfAvailableForSaleSecurities" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ProceedsFromMaturitiesPrepaymentsAndCallsOfAvailableForSaleSecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Proceeds from maturities and principal reductions of investment securities available-for-sale</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ProceedsFromMaturitiesPrepaymentsAndCallsOfAvailableForSaleSecurities" xlink:to="us-gaap_ProceedsFromMaturitiesPrepaymentsAndCallsOfAvailableForSaleSecurities_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ProceedsFromMaturitiesPrepaymentsAndCallsOfAvailableForSaleSecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Proceeds From Maturities Prepayments And Calls Of Available For Sale Securities</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ProceedsFromMaturitiesPrepaymentsAndCallsOfHeldToMaturitySecurities" xlink:type="locator" xlink:label="us-gaap_ProceedsFromMaturitiesPrepaymentsAndCallsOfHeldToMaturitySecurities" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ProceedsFromMaturitiesPrepaymentsAndCallsOfHeldToMaturitySecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Proceeds from maturities and principal reductions of investment securities held-to-maturity</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ProceedsFromMaturitiesPrepaymentsAndCallsOfHeldToMaturitySecurities" xlink:to="us-gaap_ProceedsFromMaturitiesPrepaymentsAndCallsOfHeldToMaturitySecurities_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ProceedsFromMaturitiesPrepaymentsAndCallsOfHeldToMaturitySecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Proceeds From Maturities Prepayments And Calls Of Held To Maturity Securities</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ProceedsFromSaleOfAndRedemptionsOfAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Proceeds from sales, redemptions and calls of:</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ProceedsFromSaleOfAndRedemptionsOfAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Proceeds From Sale Of And Redemptions Of [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ProceedsFromSaleOfAvailableForSaleDebtAndEquitySecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Available-for-sale investment securities</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ProceedsFromSaleOfAvailableForSaleDebtAndEquitySecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Proceeds From Sale Of Available For Sale Debt And Equity Securities</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ProceedsFromSaleOfHeldToMaturitySecurities" xlink:type="locator" xlink:label="us-gaap_ProceedsFromSaleOfHeldToMaturitySecurities" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ProceedsFromSaleOfHeldToMaturitySecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Held-to-maturity investment securities</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ProceedsFromSaleOfHeldToMaturitySecurities" xlink:to="us-gaap_ProceedsFromSaleOfHeldToMaturitySecurities_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ProceedsFromSaleOfHeldToMaturitySecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Proceeds From Sale Of Held To Maturity Securities</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ProceedsFromSaleOfForeclosedAssets" xlink:type="locator" xlink:label="us-gaap_ProceedsFromSaleOfForeclosedAssets" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ProceedsFromSaleOfForeclosedAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Real estate acquired through foreclosure</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ProceedsFromSaleOfForeclosedAssets" xlink:to="us-gaap_ProceedsFromSaleOfForeclosedAssets_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ProceedsFromSaleOfForeclosedAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Proceeds From Sale Of Foreclosed Assets</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_PaymentsForProceedsFromLoansAndLeases" xlink:type="locator" xlink:label="us-gaap_PaymentsForProceedsFromLoansAndLeases" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PaymentsForProceedsFromLoansAndLeases_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Net change in loans</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_PaymentsForProceedsFromLoansAndLeases" xlink:to="us-gaap_PaymentsForProceedsFromLoansAndLeases_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PaymentsForProceedsFromLoansAndLeases_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Payments For Proceeds From Loans And Leases</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_PaymentsToAcquirePropertyPlantAndEquipment" xlink:type="locator" xlink:label="us-gaap_PaymentsToAcquirePropertyPlantAndEquipment" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PaymentsToAcquirePropertyPlantAndEquipment_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Purchase of premises and equipment</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_PaymentsToAcquirePropertyPlantAndEquipment" xlink:to="us-gaap_PaymentsToAcquirePropertyPlantAndEquipment_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PaymentsToAcquirePropertyPlantAndEquipment_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Payments To Acquire Property Plant And Equipment</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_NetCashProvidedByUsedInInvestingActivities" xlink:type="locator" xlink:label="us-gaap_NetCashProvidedByUsedInInvestingActivities" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_NetCashProvidedByUsedInInvestingActivities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Net cash flows from investing activities</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_NetCashProvidedByUsedInInvestingActivities" xlink:to="us-gaap_NetCashProvidedByUsedInInvestingActivities_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_NetCashProvidedByUsedInInvestingActivities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Net Cash Provided By Used In Investing Activities</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_TransferOfLoansToForeclosedRealEstate" xlink:type="locator" xlink:label="pbhc_TransferOfLoansToForeclosedRealEstate" />
    <link:label xml:lang="en-US" xlink:label="pbhc_TransferOfLoansToForeclosedRealEstate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Value of loans transferred to foreclosed real estate in noncash transactions.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_TransferOfLoansToForeclosedRealEstate" xlink:to="pbhc_TransferOfLoansToForeclosedRealEstate_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_CollateralDepositsForHedgePositionIncludedInCashAndDueFromBanks" xlink:type="locator" xlink:label="pbhc_CollateralDepositsForHedgePositionIncludedInCashAndDueFromBanks" />
    <link:label xml:lang="en-US" xlink:label="pbhc_CollateralDepositsForHedgePositionIncludedInCashAndDueFromBanks_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Collateral deposits for hedge position included in cash and due from banks.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_CollateralDepositsForHedgePositionIncludedInCashAndDueFromBanks" xlink:to="pbhc_CollateralDepositsForHedgePositionIncludedInCashAndDueFromBanks_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_NetCashProvidedByUsedInOperatingActivities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Net cash flows from operating activities</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_NetCashProvidedByUsedInInvestingActivities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Net cash flows from investing activities</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_NetCashProvidedByUsedInFinancingActivitiesAbstract" xlink:type="locator" xlink:label="us-gaap_NetCashProvidedByUsedInFinancingActivitiesAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_NetCashProvidedByUsedInFinancingActivitiesAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">FINANCING ACTIVITIES</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_NetCashProvidedByUsedInFinancingActivitiesAbstract" xlink:to="us-gaap_NetCashProvidedByUsedInFinancingActivitiesAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_NetCashProvidedByUsedInFinancingActivitiesAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Net Cash Provided By Used In Financing Activities [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_IncreaseDecreaseInDemandDeposits" xlink:type="locator" xlink:label="us-gaap_IncreaseDecreaseInDemandDeposits" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncreaseDecreaseInDemandDeposits_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Net change in demand deposits, NOW accounts, savings accounts, money management deposit accounts, MMDA accounts and escrow deposits</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_IncreaseDecreaseInDemandDeposits" xlink:to="us-gaap_IncreaseDecreaseInDemandDeposits_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncreaseDecreaseInDemandDeposits_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Increase Decrease In Demand Deposits</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_IncreaseDecreaseInTimeDeposits" xlink:type="locator" xlink:label="us-gaap_IncreaseDecreaseInTimeDeposits" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncreaseDecreaseInTimeDeposits_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Net change in time deposits</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_IncreaseDecreaseInTimeDeposits" xlink:to="us-gaap_IncreaseDecreaseInTimeDeposits_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncreaseDecreaseInTimeDeposits_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Increase Decrease In Time Deposits</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_IncreaseDecreaseInBrokeredDeposits_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Net change in brokered deposits</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_IncreaseDecreaseInBrokeredDeposits_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Increase Decrease In Brokered Deposits</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ProceedsFromRepaymentsOfShortTermDebt" xlink:type="locator" xlink:label="us-gaap_ProceedsFromRepaymentsOfShortTermDebt" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ProceedsFromRepaymentsOfShortTermDebt_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Net change in short-term borrowings</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ProceedsFromRepaymentsOfShortTermDebt" xlink:to="us-gaap_ProceedsFromRepaymentsOfShortTermDebt_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ProceedsFromRepaymentsOfShortTermDebt_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Proceeds From Repayments Of Short Term Debt</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_RepaymentsOfLongTermDebt" xlink:type="locator" xlink:label="us-gaap_RepaymentsOfLongTermDebt" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RepaymentsOfLongTermDebt_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Payments on long-term borrowings</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_RepaymentsOfLongTermDebt" xlink:to="us-gaap_RepaymentsOfLongTermDebt_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RepaymentsOfLongTermDebt_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Repayments Of Long Term Debt</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ProceedsFromIssuanceOfLongTermDebt" xlink:type="locator" xlink:label="us-gaap_ProceedsFromIssuanceOfLongTermDebt" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ProceedsFromIssuanceOfLongTermDebt_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Proceeds from long-term borrowings</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ProceedsFromIssuanceOfLongTermDebt" xlink:to="us-gaap_ProceedsFromIssuanceOfLongTermDebt_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ProceedsFromIssuanceOfLongTermDebt_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Proceeds From Issuance Of Long Term Debt</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ProceedsFromStockOptionsExercised" xlink:type="locator" xlink:label="us-gaap_ProceedsFromStockOptionsExercised" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ProceedsFromStockOptionsExercised_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Proceeds from exercise of stock options</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ProceedsFromStockOptionsExercised" xlink:to="us-gaap_ProceedsFromStockOptionsExercised_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ProceedsFromStockOptionsExercised_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Proceeds From Stock Options Exercised</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ProceedsFromJuniorSubordinatedDebentureOwedToUnconsolidatedSubsidiaryTrust_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Proceeds from subordinated loan</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ProceedsFromJuniorSubordinatedDebentureOwedToUnconsolidatedSubsidiaryTrust_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Proceeds From Junior Subordinated Debenture Owed To Unconsolidated Subsidiary Trust</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_PaymentsOfDebtIssuanceCosts" xlink:type="locator" xlink:label="us-gaap_PaymentsOfDebtIssuanceCosts" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PaymentsOfDebtIssuanceCosts_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Issuance costs of subordinated loan</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_PaymentsOfDebtIssuanceCosts" xlink:to="us-gaap_PaymentsOfDebtIssuanceCosts_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PaymentsOfDebtIssuanceCosts_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Payments Of Debt Issuance Costs</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_PaymentsOfDividendsCommonStock" xlink:type="locator" xlink:label="us-gaap_PaymentsOfDividendsCommonStock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PaymentsOfDividendsCommonStock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Cash dividends paid to common shareholders</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_PaymentsOfDividendsCommonStock" xlink:to="us-gaap_PaymentsOfDividendsCommonStock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PaymentsOfDividendsCommonStock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Payments Of Dividends Common Stock</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_PaymentsOfDividendsPreferredStockAndPreferenceStock" xlink:type="locator" xlink:label="us-gaap_PaymentsOfDividendsPreferredStockAndPreferenceStock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PaymentsOfDividendsPreferredStockAndPreferenceStock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Cash dividends paid to preferred shareholders</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_PaymentsOfDividendsPreferredStockAndPreferenceStock" xlink:to="us-gaap_PaymentsOfDividendsPreferredStockAndPreferenceStock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PaymentsOfDividendsPreferredStockAndPreferenceStock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Payments Of Dividends Preferred Stock And Preference Stock</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_PaymentsOfDividendsWarrants_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Cash dividends paid on warrants</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_PaymentsOfDividendsWarrants_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Payments Of Dividends Warrants</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ProceedsFromIssuanceOfPrivatePlacement" xlink:type="locator" xlink:label="us-gaap_ProceedsFromIssuanceOfPrivatePlacement" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ProceedsFromIssuanceOfPrivatePlacement_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Net proceeds from private placement</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ProceedsFromIssuanceOfPrivatePlacement" xlink:to="us-gaap_ProceedsFromIssuanceOfPrivatePlacement_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ProceedsFromIssuanceOfPrivatePlacement_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Proceeds From Issuance Of Private Placement</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ProceedsFromFinanceLeaseTransaction_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Proceeds from finance lease transaction</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ProceedsFromFinanceLeaseTransaction_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Proceeds From Finance Lease Transaction</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ProceedsFromPaymentsToMinorityShareholders" xlink:type="locator" xlink:label="us-gaap_ProceedsFromPaymentsToMinorityShareholders" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ProceedsFromPaymentsToMinorityShareholders_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Change in noncontrolling interest, net</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ProceedsFromPaymentsToMinorityShareholders" xlink:to="us-gaap_ProceedsFromPaymentsToMinorityShareholders_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ProceedsFromPaymentsToMinorityShareholders_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Proceeds From Payments To Minority Shareholders</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_NetCashProvidedByUsedInFinancingActivities" xlink:type="locator" xlink:label="us-gaap_NetCashProvidedByUsedInFinancingActivities" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_NetCashProvidedByUsedInFinancingActivities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Net cash flows from financing activities</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_NetCashProvidedByUsedInFinancingActivities" xlink:to="us-gaap_NetCashProvidedByUsedInFinancingActivities_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_NetCashProvidedByUsedInFinancingActivities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Net Cash Provided By Used In Financing Activities</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalentsPeriodIncreaseDecreaseIncludingExchangeRateEffect" xlink:type="locator" xlink:label="us-gaap_CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalentsPeriodIncreaseDecreaseIncludingExchangeRateEffect" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalentsPeriodIncreaseDecreaseIncludingExchangeRateEffect_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Change in cash and cash equivalents</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalentsPeriodIncreaseDecreaseIncludingExchangeRateEffect" xlink:to="us-gaap_CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalentsPeriodIncreaseDecreaseIncludingExchangeRateEffect_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalentsPeriodIncreaseDecreaseIncludingExchangeRateEffect_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Cash Cash Equivalents Restricted Cash And Restricted Cash Equivalents Period Increase Decrease Including Exchange Rate Effect</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalentsIncludingDisposalGroupAndDiscontinuedOperations" xlink:type="locator" xlink:label="us-gaap_CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalentsIncludingDisposalGroupAndDiscontinuedOperations" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalentsIncludingDisposalGroupAndDiscontinuedOperations_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/periodStartLabel">Cash and cash equivalents at beginning of period</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalentsIncludingDisposalGroupAndDiscontinuedOperations" xlink:to="us-gaap_CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalentsIncludingDisposalGroupAndDiscontinuedOperations_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalentsIncludingDisposalGroupAndDiscontinuedOperations_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Cash Cash Equivalents Restricted Cash And Restricted Cash Equivalents Including Disposal Group And Discontinued Operations</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalentsIncludingDisposalGroupAndDiscontinuedOperations_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/periodEndLabel">Cash and cash equivalents at end of period</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_SupplementalCashFlowInformationAbstract" xlink:type="locator" xlink:label="us-gaap_SupplementalCashFlowInformationAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SupplementalCashFlowInformationAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">CASH PAID DURING THE PERIOD FOR:</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_SupplementalCashFlowInformationAbstract" xlink:to="us-gaap_SupplementalCashFlowInformationAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SupplementalCashFlowInformationAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Supplemental Cash Flow Information [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_InterestPaidNet" xlink:type="locator" xlink:label="us-gaap_InterestPaidNet" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestPaidNet_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Interest</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_InterestPaidNet" xlink:to="us-gaap_InterestPaidNet_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestPaidNet_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Interest Paid Net</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_IncomeTaxesPaid" xlink:type="locator" xlink:label="us-gaap_IncomeTaxesPaid" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncomeTaxesPaid_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Income taxes</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_IncomeTaxesPaid" xlink:to="us-gaap_IncomeTaxesPaid_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncomeTaxesPaid_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Income Taxes Paid</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_OtherNoncashInvestingAndFinancingItemsAbstract" xlink:type="locator" xlink:label="us-gaap_OtherNoncashInvestingAndFinancingItemsAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherNoncashInvestingAndFinancingItemsAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">NON-CASH INVESTING ACTIVITY</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_OtherNoncashInvestingAndFinancingItemsAbstract" xlink:to="us-gaap_OtherNoncashInvestingAndFinancingItemsAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherNoncashInvestingAndFinancingItemsAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Other Noncash Investing And Financing Items [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_TransferOfLoansToForeclosedRealEstate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Real estate acquired in exchange for loans</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_TransferOfLoansToForeclosedRealEstate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Transfer Of Loans To Foreclosed Real Estate</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_RestrictedCashAbstract" xlink:type="locator" xlink:label="us-gaap_RestrictedCashAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RestrictedCashAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">RESTRICTED CASH</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_RestrictedCashAbstract" xlink:to="us-gaap_RestrictedCashAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RestrictedCashAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Restricted Cash [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_CollateralDepositsForHedgePositionIncludedInCashAndDueFromBanks_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Collateral deposits for hedge position included in cash and due from banks</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_CollateralDepositsForHedgePositionIncludedInCashAndDueFromBanks_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Collateral Deposits For Hedge Position Included In Cash And Due From Banks</link:label>
    <link:loc xlink:href="https://xbrl.sec.gov/dei/2020/dei-2020-01-31.xsd#dei_DocumentsIncorporatedByReferenceTextBlock" xlink:type="locator" xlink:label="dei_DocumentsIncorporatedByReferenceTextBlock" />
    <link:label xml:lang="en-US" xlink:label="dei_DocumentsIncorporatedByReferenceTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Documents Incorporated by Reference</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="dei_DocumentsIncorporatedByReferenceTextBlock" xlink:to="dei_DocumentsIncorporatedByReferenceTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="dei_DocumentsIncorporatedByReferenceTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Documents Incorporated By Reference [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AccountingPoliciesAbstract" xlink:type="locator" xlink:label="us-gaap_AccountingPoliciesAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AccountingPoliciesAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Accounting Policies [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AccountingPoliciesAbstract" xlink:to="us-gaap_AccountingPoliciesAbstract_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_SignificantAccountingPoliciesTextBlock" xlink:type="locator" xlink:label="us-gaap_SignificantAccountingPoliciesTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SignificantAccountingPoliciesTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">SUMMARY OF SIGNIFICANT ACCOUNTING POLICIES</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_SignificantAccountingPoliciesTextBlock" xlink:to="us-gaap_SignificantAccountingPoliciesTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SignificantAccountingPoliciesTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Significant Accounting Policies [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_NewAccountingPronouncementsAndChangesInAccountingPrinciplesAbstract" xlink:type="locator" xlink:label="us-gaap_NewAccountingPronouncementsAndChangesInAccountingPrinciplesAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_NewAccountingPronouncementsAndChangesInAccountingPrinciplesAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">New Accounting Pronouncements And Changes In Accounting Principles [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_NewAccountingPronouncementsAndChangesInAccountingPrinciplesAbstract" xlink:to="us-gaap_NewAccountingPronouncementsAndChangesInAccountingPrinciplesAbstract_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DescriptionOfNewAccountingPronouncementsNotYetAdopted" xlink:type="locator" xlink:label="us-gaap_DescriptionOfNewAccountingPronouncementsNotYetAdopted" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DescriptionOfNewAccountingPronouncementsNotYetAdopted_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">NEW ACCOUNTING PRONOUNCEMENTS</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DescriptionOfNewAccountingPronouncementsNotYetAdopted" xlink:to="us-gaap_DescriptionOfNewAccountingPronouncementsNotYetAdopted_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DescriptionOfNewAccountingPronouncementsNotYetAdopted_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Description Of New Accounting Pronouncements Not Yet Adopted</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_EarningsPerShareAbstract" xlink:type="locator" xlink:label="us-gaap_EarningsPerShareAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EarningsPerShareAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Earnings Per Share [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_EarningsPerShareAbstract" xlink:to="us-gaap_EarningsPerShareAbstract_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_EarningsPerShareTextBlock" xlink:type="locator" xlink:label="us-gaap_EarningsPerShareTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EarningsPerShareTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Earnings Per Share</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_EarningsPerShareTextBlock" xlink:to="us-gaap_EarningsPerShareTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EarningsPerShareTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Earnings Per Share [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_InvestmentsDebtAndEquitySecuritiesAbstract" xlink:type="locator" xlink:label="us-gaap_InvestmentsDebtAndEquitySecuritiesAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InvestmentsDebtAndEquitySecuritiesAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Investments Debt And Equity Securities [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_InvestmentsDebtAndEquitySecuritiesAbstract" xlink:to="us-gaap_InvestmentsDebtAndEquitySecuritiesAbstract_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_InvestmentsInDebtAndMarketableEquitySecuritiesAndCertainTradingAssetsDisclosureTextBlock" xlink:type="locator" xlink:label="us-gaap_InvestmentsInDebtAndMarketableEquitySecuritiesAndCertainTradingAssetsDisclosureTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InvestmentsInDebtAndMarketableEquitySecuritiesAndCertainTradingAssetsDisclosureTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Investment Securities</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_InvestmentsInDebtAndMarketableEquitySecuritiesAndCertainTradingAssetsDisclosureTextBlock" xlink:to="us-gaap_InvestmentsInDebtAndMarketableEquitySecuritiesAndCertainTradingAssetsDisclosureTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InvestmentsInDebtAndMarketableEquitySecuritiesAndCertainTradingAssetsDisclosureTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Investments In Debt And Marketable Equity Securities And Certain Trading Assets Disclosure [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ReceivablesAbstract" xlink:type="locator" xlink:label="us-gaap_ReceivablesAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ReceivablesAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Receivables [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ReceivablesAbstract" xlink:to="us-gaap_ReceivablesAbstract_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LoansNotesTradeAndOtherReceivablesDisclosureTextBlock" xlink:type="locator" xlink:label="us-gaap_LoansNotesTradeAndOtherReceivablesDisclosureTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LoansNotesTradeAndOtherReceivablesDisclosureTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">LOANS</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LoansNotesTradeAndOtherReceivablesDisclosureTextBlock" xlink:to="us-gaap_LoansNotesTradeAndOtherReceivablesDisclosureTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LoansNotesTradeAndOtherReceivablesDisclosureTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Loans Notes Trade And Other Receivables Disclosure [Text Block]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_AllowanceForLoanLossesAbstract" xlink:type="locator" xlink:label="pbhc_AllowanceForLoanLossesAbstract" />
    <link:label xml:lang="en-US" xlink:label="pbhc_AllowanceForLoanLossesAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Allowance for loan losses.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_AllowanceForLoanLossesAbstract" xlink:to="pbhc_AllowanceForLoanLossesAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_AllowanceForLoanLossesAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Allowance For Loan Losses [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AllowanceForCreditLossesTextBlock" xlink:type="locator" xlink:label="us-gaap_AllowanceForCreditLossesTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AllowanceForCreditLossesTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">ALLOWANCE FOR LOAN LOSSES</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AllowanceForCreditLossesTextBlock" xlink:to="us-gaap_AllowanceForCreditLossesTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AllowanceForCreditLossesTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Allowance For Credit Losses [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ServicingAssetAbstract" xlink:type="locator" xlink:label="us-gaap_ServicingAssetAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ServicingAssetAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Servicing Asset [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ServicingAssetAbstract" xlink:to="us-gaap_ServicingAssetAbstract_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_TransfersAndServicingOfFinancialAssetsTextBlock" xlink:type="locator" xlink:label="us-gaap_TransfersAndServicingOfFinancialAssetsTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TransfersAndServicingOfFinancialAssetsTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">SERVICING</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_TransfersAndServicingOfFinancialAssetsTextBlock" xlink:to="us-gaap_TransfersAndServicingOfFinancialAssetsTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TransfersAndServicingOfFinancialAssetsTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Transfers And Servicing Of Financial Assets [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_PropertyPlantAndEquipmentAbstract" xlink:type="locator" xlink:label="us-gaap_PropertyPlantAndEquipmentAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PropertyPlantAndEquipmentAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Property Plant And Equipment [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_PropertyPlantAndEquipmentAbstract" xlink:to="us-gaap_PropertyPlantAndEquipmentAbstract_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_PropertyPlantAndEquipmentDisclosureTextBlock" xlink:type="locator" xlink:label="us-gaap_PropertyPlantAndEquipmentDisclosureTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PropertyPlantAndEquipmentDisclosureTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Premises and Equipment</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_PropertyPlantAndEquipmentDisclosureTextBlock" xlink:to="us-gaap_PropertyPlantAndEquipmentDisclosureTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PropertyPlantAndEquipmentDisclosureTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Property Plant And Equipment Disclosure [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_RealEstateOwnedDisclosureOfDetailedComponentsAbstract" xlink:type="locator" xlink:label="us-gaap_RealEstateOwnedDisclosureOfDetailedComponentsAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RealEstateOwnedDisclosureOfDetailedComponentsAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Real Estate Owned Disclosure Of Detailed Components [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_RealEstateOwnedDisclosureOfDetailedComponentsAbstract" xlink:to="us-gaap_RealEstateOwnedDisclosureOfDetailedComponentsAbstract_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_RealEstateOwnedTextBlock" xlink:type="locator" xlink:label="us-gaap_RealEstateOwnedTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RealEstateOwnedTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Foreclosed Real Estate</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_RealEstateOwnedTextBlock" xlink:to="us-gaap_RealEstateOwnedTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RealEstateOwnedTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Real Estate Owned [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_GoodwillAndIntangibleAssetsDisclosureAbstract" xlink:type="locator" xlink:label="us-gaap_GoodwillAndIntangibleAssetsDisclosureAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_GoodwillAndIntangibleAssetsDisclosureAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Goodwill And Intangible Assets Disclosure [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_GoodwillAndIntangibleAssetsDisclosureAbstract" xlink:to="us-gaap_GoodwillAndIntangibleAssetsDisclosureAbstract_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_GoodwillDisclosureTextBlock" xlink:type="locator" xlink:label="us-gaap_GoodwillDisclosureTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_GoodwillDisclosureTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">GOODWILL AND INTANGIBLE ASSETS</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_GoodwillDisclosureTextBlock" xlink:to="us-gaap_GoodwillDisclosureTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_GoodwillDisclosureTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Goodwill Disclosure [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DepositLiabilitiesDisclosuresTextBlock" xlink:type="locator" xlink:label="us-gaap_DepositLiabilitiesDisclosuresTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DepositLiabilitiesDisclosuresTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">DEPOSITS</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DepositLiabilitiesDisclosuresTextBlock" xlink:to="us-gaap_DepositLiabilitiesDisclosuresTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DepositLiabilitiesDisclosuresTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deposit Liabilities Disclosures [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DebtDisclosureAbstract" xlink:type="locator" xlink:label="us-gaap_DebtDisclosureAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtDisclosureAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Debt Disclosure [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DebtDisclosureAbstract" xlink:to="us-gaap_DebtDisclosureAbstract_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DebtDisclosureTextBlock" xlink:type="locator" xlink:label="us-gaap_DebtDisclosureTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtDisclosureTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">BORROWED FUNDS</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DebtDisclosureTextBlock" xlink:to="us-gaap_DebtDisclosureTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtDisclosureTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Debt Disclosure [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_SubordinatedBorrowingsAbstract" xlink:type="locator" xlink:label="us-gaap_SubordinatedBorrowingsAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SubordinatedBorrowingsAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Subordinated Borrowings [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_SubordinatedBorrowingsAbstract" xlink:to="us-gaap_SubordinatedBorrowingsAbstract_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_SubordinatedBorrowingsDisclosureTextBlock" xlink:type="locator" xlink:label="us-gaap_SubordinatedBorrowingsDisclosureTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SubordinatedBorrowingsDisclosureTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">SUBORDINATED LOANS</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_SubordinatedBorrowingsDisclosureTextBlock" xlink:to="us-gaap_SubordinatedBorrowingsDisclosureTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SubordinatedBorrowingsDisclosureTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Subordinated Borrowings Disclosure [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_CompensationAndRetirementDisclosureAbstract" xlink:type="locator" xlink:label="us-gaap_CompensationAndRetirementDisclosureAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CompensationAndRetirementDisclosureAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Compensation And Retirement Disclosure [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_CompensationAndRetirementDisclosureAbstract" xlink:to="us-gaap_CompensationAndRetirementDisclosureAbstract_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_PensionAndOtherPostretirementBenefitsDisclosureTextBlock" xlink:type="locator" xlink:label="us-gaap_PensionAndOtherPostretirementBenefitsDisclosureTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PensionAndOtherPostretirementBenefitsDisclosureTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">EMPLOYEE BENEFITS AND DEFERRED COMPENSATION AND SUPPLEMENTAL RETIREMENT PLANS</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_PensionAndOtherPostretirementBenefitsDisclosureTextBlock" xlink:to="us-gaap_PensionAndOtherPostretirementBenefitsDisclosureTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PensionAndOtherPostretirementBenefitsDisclosureTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Pension And Other Postretirement Benefits Disclosure [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DisclosureOfCompensationRelatedCostsSharebasedPaymentsAbstract" xlink:type="locator" xlink:label="us-gaap_DisclosureOfCompensationRelatedCostsSharebasedPaymentsAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DisclosureOfCompensationRelatedCostsSharebasedPaymentsAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Disclosure Of Compensation Related Costs Sharebased Payments [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DisclosureOfCompensationRelatedCostsSharebasedPaymentsAbstract" xlink:to="us-gaap_DisclosureOfCompensationRelatedCostsSharebasedPaymentsAbstract_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DisclosureOfCompensationRelatedCostsShareBasedPaymentsTextBlock" xlink:type="locator" xlink:label="us-gaap_DisclosureOfCompensationRelatedCostsShareBasedPaymentsTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DisclosureOfCompensationRelatedCostsShareBasedPaymentsTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">STOCK BASED COMPENSATION PLANS</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DisclosureOfCompensationRelatedCostsShareBasedPaymentsTextBlock" xlink:to="us-gaap_DisclosureOfCompensationRelatedCostsShareBasedPaymentsTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DisclosureOfCompensationRelatedCostsShareBasedPaymentsTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Disclosure Of Compensation Related Costs Share Based Payments [Text Block]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_EmployeeStockOwnershipPlanTextBlock" xlink:type="locator" xlink:label="pbhc_EmployeeStockOwnershipPlanTextBlock" />
    <link:label xml:lang="en-US" xlink:label="pbhc_EmployeeStockOwnershipPlanTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The entire disclosure of employee stock ownership plan.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_EmployeeStockOwnershipPlanTextBlock" xlink:to="pbhc_EmployeeStockOwnershipPlanTextBlock_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_EmployeeStockOwnershipPlanAbstract" xlink:type="locator" xlink:label="pbhc_EmployeeStockOwnershipPlanAbstract" />
    <link:label xml:lang="en-US" xlink:label="pbhc_EmployeeStockOwnershipPlanAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">EMPLOYEE STOCK OWNERSHIP PLAN [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_EmployeeStockOwnershipPlanAbstract" xlink:to="pbhc_EmployeeStockOwnershipPlanAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_EmployeeStockOwnershipPlanAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Employee Stock Ownership Plan [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_EmployeeStockOwnershipPlanTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">EMPLOYEE STOCK OWNERSHIP PLAN</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_EmployeeStockOwnershipPlanTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Employee Stock Ownership Plan [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_IncomeTaxDisclosureAbstract" xlink:type="locator" xlink:label="us-gaap_IncomeTaxDisclosureAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncomeTaxDisclosureAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Income Tax Disclosure [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_IncomeTaxDisclosureAbstract" xlink:to="us-gaap_IncomeTaxDisclosureAbstract_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_IncomeTaxDisclosureTextBlock" xlink:type="locator" xlink:label="us-gaap_IncomeTaxDisclosureTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncomeTaxDisclosureTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">INCOME TAXES</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_IncomeTaxDisclosureTextBlock" xlink:to="us-gaap_IncomeTaxDisclosureTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncomeTaxDisclosureTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Income Tax Disclosure [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_CommitmentsAndContingenciesDisclosureAbstract" xlink:type="locator" xlink:label="us-gaap_CommitmentsAndContingenciesDisclosureAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CommitmentsAndContingenciesDisclosureAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Commitments And Contingencies Disclosure [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_CommitmentsAndContingenciesDisclosureAbstract" xlink:to="us-gaap_CommitmentsAndContingenciesDisclosureAbstract_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_CommitmentsAndContingenciesDisclosureTextBlock" xlink:type="locator" xlink:label="us-gaap_CommitmentsAndContingenciesDisclosureTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CommitmentsAndContingenciesDisclosureTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">COMMITMENTS AND CONTINGENCIES</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_CommitmentsAndContingenciesDisclosureTextBlock" xlink:to="us-gaap_CommitmentsAndContingenciesDisclosureTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CommitmentsAndContingenciesDisclosureTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Commitments And Contingencies Disclosure [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DisclosureOfRestrictionsOnDividendsLoansAndAdvancesDisclosureAbstract" xlink:type="locator" xlink:label="us-gaap_DisclosureOfRestrictionsOnDividendsLoansAndAdvancesDisclosureAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DisclosureOfRestrictionsOnDividendsLoansAndAdvancesDisclosureAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Disclosure Of Restrictions On Dividends Loans And Advances Disclosure [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DisclosureOfRestrictionsOnDividendsLoansAndAdvancesDisclosureAbstract" xlink:to="us-gaap_DisclosureOfRestrictionsOnDividendsLoansAndAdvancesDisclosureAbstract_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_RestrictionsOnDividendsLoansAndAdvancesTextBlock" xlink:type="locator" xlink:label="us-gaap_RestrictionsOnDividendsLoansAndAdvancesTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RestrictionsOnDividendsLoansAndAdvancesTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">DIVIDENDS AND RESTRICTIONS</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_RestrictionsOnDividendsLoansAndAdvancesTextBlock" xlink:to="us-gaap_RestrictionsOnDividendsLoansAndAdvancesTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RestrictionsOnDividendsLoansAndAdvancesTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Restrictions On Dividends Loans And Advances [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/srt/2020/elts/srt-2020-01-31.xsd#srt_RegulatoryCapitalRequirementsAbstract" xlink:type="locator" xlink:label="srt_RegulatoryCapitalRequirementsAbstract" />
    <link:label xml:lang="en-US" xlink:label="srt_RegulatoryCapitalRequirementsAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Regulatory Capital Requirements [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="srt_RegulatoryCapitalRequirementsAbstract" xlink:to="srt_RegulatoryCapitalRequirementsAbstract_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_RegulatoryCapitalRequirementsUnderBankingRegulationsTextBlock" xlink:type="locator" xlink:label="us-gaap_RegulatoryCapitalRequirementsUnderBankingRegulationsTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RegulatoryCapitalRequirementsUnderBankingRegulationsTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">REGULATORY MATTERS</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_RegulatoryCapitalRequirementsUnderBankingRegulationsTextBlock" xlink:to="us-gaap_RegulatoryCapitalRequirementsUnderBankingRegulationsTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RegulatoryCapitalRequirementsUnderBankingRegulationsTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Regulatory Capital Requirements Under Banking Regulations [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DerivativeInstrumentsAndHedgingActivitiesDisclosureAbstract" xlink:type="locator" xlink:label="us-gaap_DerivativeInstrumentsAndHedgingActivitiesDisclosureAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DerivativeInstrumentsAndHedgingActivitiesDisclosureAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Derivative Instruments And Hedging Activities Disclosure [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DerivativeInstrumentsAndHedgingActivitiesDisclosureAbstract" xlink:to="us-gaap_DerivativeInstrumentsAndHedgingActivitiesDisclosureAbstract_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DerivativeInstrumentsAndHedgingActivitiesDisclosureTextBlock" xlink:type="locator" xlink:label="us-gaap_DerivativeInstrumentsAndHedgingActivitiesDisclosureTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DerivativeInstrumentsAndHedgingActivitiesDisclosureTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">INTEREST RATE DERIVATIVE</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DerivativeInstrumentsAndHedgingActivitiesDisclosureTextBlock" xlink:to="us-gaap_DerivativeInstrumentsAndHedgingActivitiesDisclosureTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DerivativeInstrumentsAndHedgingActivitiesDisclosureTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Derivative Instruments And Hedging Activities Disclosure [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FairValueDisclosuresAbstract" xlink:type="locator" xlink:label="us-gaap_FairValueDisclosuresAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueDisclosuresAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Fair Value Disclosures [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FairValueDisclosuresAbstract" xlink:to="us-gaap_FairValueDisclosuresAbstract_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FairValueDisclosuresTextBlock" xlink:type="locator" xlink:label="us-gaap_FairValueDisclosuresTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueDisclosuresTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">FAIR VALUE MEASUREMENTS AND DISCLOSURES</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FairValueDisclosuresTextBlock" xlink:to="us-gaap_FairValueDisclosuresTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueDisclosuresTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Fair Value Disclosures [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/srt/2020/elts/srt-2020-01-31.xsd#srt_CondensedFinancialInformationOfParentCompanyOnlyDisclosureAbstract" xlink:type="locator" xlink:label="srt_CondensedFinancialInformationOfParentCompanyOnlyDisclosureAbstract" />
    <link:label xml:lang="en-US" xlink:label="srt_CondensedFinancialInformationOfParentCompanyOnlyDisclosureAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Condensed Financial Information Of Parent Company Only Disclosure [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="srt_CondensedFinancialInformationOfParentCompanyOnlyDisclosureAbstract" xlink:to="srt_CondensedFinancialInformationOfParentCompanyOnlyDisclosureAbstract_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_CondensedFinancialInformationOfParentCompanyOnlyDisclosureTextBlock" xlink:type="locator" xlink:label="us-gaap_CondensedFinancialInformationOfParentCompanyOnlyDisclosureTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CondensedFinancialInformationOfParentCompanyOnlyDisclosureTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">PARENT COMPANY - FINANCIAL INFORMATION</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_CondensedFinancialInformationOfParentCompanyOnlyDisclosureTextBlock" xlink:to="us-gaap_CondensedFinancialInformationOfParentCompanyOnlyDisclosureTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CondensedFinancialInformationOfParentCompanyOnlyDisclosureTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Condensed Financial Information Of Parent Company Only Disclosure [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_RelatedPartyTransactionsAbstract" xlink:type="locator" xlink:label="us-gaap_RelatedPartyTransactionsAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RelatedPartyTransactionsAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Related Party Transactions [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_RelatedPartyTransactionsAbstract" xlink:to="us-gaap_RelatedPartyTransactionsAbstract_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_RelatedPartyTransactionsDisclosureTextBlock" xlink:type="locator" xlink:label="us-gaap_RelatedPartyTransactionsDisclosureTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RelatedPartyTransactionsDisclosureTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">RELATED PARTY TRANSACTIONS</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_RelatedPartyTransactionsDisclosureTextBlock" xlink:to="us-gaap_RelatedPartyTransactionsDisclosureTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RelatedPartyTransactionsDisclosureTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Related Party Transactions Disclosure [Text Block]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ConversionAndReorganizationTextBlock" xlink:type="locator" xlink:label="pbhc_ConversionAndReorganizationTextBlock" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ConversionAndReorganizationTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Refers to conversion and reorganization.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ConversionAndReorganizationTextBlock" xlink:to="pbhc_ConversionAndReorganizationTextBlock_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ConversionAndReorganizationAbstract" xlink:type="locator" xlink:label="pbhc_ConversionAndReorganizationAbstract" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ConversionAndReorganizationAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Conversion and reorganization.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ConversionAndReorganizationAbstract" xlink:to="pbhc_ConversionAndReorganizationAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ConversionAndReorganizationAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Conversion And Reorganization [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ConversionAndReorganizationTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">CONVERSION AND REORGANIZATION</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ConversionAndReorganizationTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Conversion And Reorganization [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AccumulatedOtherComprehensiveIncomeLossNetOfTaxAbstract" xlink:type="locator" xlink:label="us-gaap_AccumulatedOtherComprehensiveIncomeLossNetOfTaxAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AccumulatedOtherComprehensiveIncomeLossNetOfTaxAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Accumulated Other Comprehensive Income Loss Net Of Tax [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AccumulatedOtherComprehensiveIncomeLossNetOfTaxAbstract" xlink:to="us-gaap_AccumulatedOtherComprehensiveIncomeLossNetOfTaxAbstract_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ComprehensiveIncomeNoteTextBlock" xlink:type="locator" xlink:label="us-gaap_ComprehensiveIncomeNoteTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ComprehensiveIncomeNoteTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">ACCUMULATED OTHER COMPREHENSIVE INCOME (LOSS)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ComprehensiveIncomeNoteTextBlock" xlink:to="us-gaap_ComprehensiveIncomeNoteTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ComprehensiveIncomeNoteTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Comprehensive Income Note [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_RevenueFromContractWithCustomerAbstract" xlink:type="locator" xlink:label="us-gaap_RevenueFromContractWithCustomerAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RevenueFromContractWithCustomerAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Revenue From Contract With Customer [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_RevenueFromContractWithCustomerAbstract" xlink:to="us-gaap_RevenueFromContractWithCustomerAbstract_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_RevenueFromContractWithCustomerTextBlock" xlink:type="locator" xlink:label="us-gaap_RevenueFromContractWithCustomerTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RevenueFromContractWithCustomerTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Noninterest Income</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_RevenueFromContractWithCustomerTextBlock" xlink:to="us-gaap_RevenueFromContractWithCustomerTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RevenueFromContractWithCustomerTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Revenue From Contract With Customer [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LeasesAbstract" xlink:type="locator" xlink:label="us-gaap_LeasesAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LeasesAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Leases [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LeasesAbstract" xlink:to="us-gaap_LeasesAbstract_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LesseeOperatingLeasesTextBlock" xlink:type="locator" xlink:label="us-gaap_LesseeOperatingLeasesTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LesseeOperatingLeasesTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">LEASES</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LesseeOperatingLeasesTextBlock" xlink:to="us-gaap_LesseeOperatingLeasesTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LesseeOperatingLeasesTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Lessee Operating Leases [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ExtraordinaryAndUnusualItemsAbstract" xlink:type="locator" xlink:label="us-gaap_ExtraordinaryAndUnusualItemsAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ExtraordinaryAndUnusualItemsAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Extraordinary And Unusual Items [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ExtraordinaryAndUnusualItemsAbstract" xlink:to="us-gaap_ExtraordinaryAndUnusualItemsAbstract_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_UnusualOrInfrequentItemsDisclosureTextBlock" xlink:type="locator" xlink:label="us-gaap_UnusualOrInfrequentItemsDisclosureTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_UnusualOrInfrequentItemsDisclosureTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">COVID-19</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_UnusualOrInfrequentItemsDisclosureTextBlock" xlink:to="us-gaap_UnusualOrInfrequentItemsDisclosureTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_UnusualOrInfrequentItemsDisclosureTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Unusual Or Infrequent Items Disclosure [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_SubsequentEventsAbstract" xlink:type="locator" xlink:label="us-gaap_SubsequentEventsAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SubsequentEventsAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Subsequent Events [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_SubsequentEventsAbstract" xlink:to="us-gaap_SubsequentEventsAbstract_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_SubsequentEventsTextBlock" xlink:type="locator" xlink:label="us-gaap_SubsequentEventsTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SubsequentEventsTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Subsequent Events</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_SubsequentEventsTextBlock" xlink:to="us-gaap_SubsequentEventsTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SubsequentEventsTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Subsequent Events [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ConsolidationPolicyTextBlock" xlink:type="locator" xlink:label="us-gaap_ConsolidationPolicyTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ConsolidationPolicyTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Nature of Operations</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ConsolidationPolicyTextBlock" xlink:to="us-gaap_ConsolidationPolicyTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ConsolidationPolicyTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Consolidation Policy [Text Block]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ExtraordinaryAndUnusualItemsPolicyTextBlock" xlink:type="locator" xlink:label="pbhc_ExtraordinaryAndUnusualItemsPolicyTextBlock" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ExtraordinaryAndUnusualItemsPolicyTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Extraordinary and unusual items.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ExtraordinaryAndUnusualItemsPolicyTextBlock" xlink:to="pbhc_ExtraordinaryAndUnusualItemsPolicyTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ExtraordinaryAndUnusualItemsPolicyTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Discussion of COVID-19 Pandemic</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ExtraordinaryAndUnusualItemsPolicyTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Extraordinary And Unusual Items Policy [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_UseOfEstimates" xlink:type="locator" xlink:label="us-gaap_UseOfEstimates" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_UseOfEstimates_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Use of Estimates in the Preparation of Consolidated Financial Statements</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_UseOfEstimates" xlink:to="us-gaap_UseOfEstimates_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_UseOfEstimates_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Use Of Estimates</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ConcentrationRiskCreditRisk" xlink:type="locator" xlink:label="us-gaap_ConcentrationRiskCreditRisk" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ConcentrationRiskCreditRisk_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Significant Group Concentrations of Credit Risk</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ConcentrationRiskCreditRisk" xlink:to="us-gaap_ConcentrationRiskCreditRisk_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ConcentrationRiskCreditRisk_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Concentration Risk Credit Risk</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AdvertisingCostsPolicyTextBlock" xlink:type="locator" xlink:label="us-gaap_AdvertisingCostsPolicyTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AdvertisingCostsPolicyTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Advertising</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AdvertisingCostsPolicyTextBlock" xlink:to="us-gaap_AdvertisingCostsPolicyTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AdvertisingCostsPolicyTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Advertising Costs Policy [Text Block]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_NoncontrollingInterestPolicyTextBlock" xlink:type="locator" xlink:label="pbhc_NoncontrollingInterestPolicyTextBlock" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NoncontrollingInterestPolicyTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Disclosure of accounting policy treatment of Noncontrolling Interest.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_NoncontrollingInterestPolicyTextBlock" xlink:to="pbhc_NoncontrollingInterestPolicyTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NoncontrollingInterestPolicyTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Noncontrolling Interest</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NoncontrollingInterestPolicyTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Noncontrolling Interest Policy [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_CashAndCashEquivalentsPolicyTextBlock" xlink:type="locator" xlink:label="us-gaap_CashAndCashEquivalentsPolicyTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CashAndCashEquivalentsPolicyTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Cash and Cash Equivalents</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_CashAndCashEquivalentsPolicyTextBlock" xlink:to="us-gaap_CashAndCashEquivalentsPolicyTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CashAndCashEquivalentsPolicyTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Cash And Cash Equivalents Policy [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_InvestmentPolicyTextBlock" xlink:type="locator" xlink:label="us-gaap_InvestmentPolicyTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InvestmentPolicyTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Investment Securities</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_InvestmentPolicyTextBlock" xlink:to="us-gaap_InvestmentPolicyTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InvestmentPolicyTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Investment Policy [Text Block]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_FederalHomeLoanBankFhlbStockPolicyTextBlock" xlink:type="locator" xlink:label="pbhc_FederalHomeLoanBankFhlbStockPolicyTextBlock" />
    <link:label xml:lang="en-US" xlink:label="pbhc_FederalHomeLoanBankFhlbStockPolicyTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Disclosure of accounting policy treatment of Federal Home Loan Bank (FHLB) Stock.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_FederalHomeLoanBankFhlbStockPolicyTextBlock" xlink:to="pbhc_FederalHomeLoanBankFhlbStockPolicyTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_FederalHomeLoanBankFhlbStockPolicyTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Federal Home Loan Bank Stock</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_FederalHomeLoanBankFhlbStockPolicyTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Federal Home Loan Bank Fhlb Stock Policy [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_TransfersAndServicingOfFinancialAssetsPolicyTextBlock" xlink:type="locator" xlink:label="us-gaap_TransfersAndServicingOfFinancialAssetsPolicyTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TransfersAndServicingOfFinancialAssetsPolicyTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Transfers of Financial Assets</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_TransfersAndServicingOfFinancialAssetsPolicyTextBlock" xlink:to="us-gaap_TransfersAndServicingOfFinancialAssetsPolicyTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TransfersAndServicingOfFinancialAssetsPolicyTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Transfers And Servicing Of Financial Assets Policy [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FinanceLoanAndLeaseReceivablesHeldForInvestmentPolicy" xlink:type="locator" xlink:label="us-gaap_FinanceLoanAndLeaseReceivablesHeldForInvestmentPolicy" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinanceLoanAndLeaseReceivablesHeldForInvestmentPolicy_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Loans</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FinanceLoanAndLeaseReceivablesHeldForInvestmentPolicy" xlink:to="us-gaap_FinanceLoanAndLeaseReceivablesHeldForInvestmentPolicy_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinanceLoanAndLeaseReceivablesHeldForInvestmentPolicy_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Finance Loan And Lease Receivables Held For Investment Policy</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LoansAndLeasesReceivableAllowanceForLoanLossesPolicy" xlink:type="locator" xlink:label="us-gaap_LoansAndLeasesReceivableAllowanceForLoanLossesPolicy" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LoansAndLeasesReceivableAllowanceForLoanLossesPolicy_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Allowance for Loan Losses</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LoansAndLeasesReceivableAllowanceForLoanLossesPolicy" xlink:to="us-gaap_LoansAndLeasesReceivableAllowanceForLoanLossesPolicy_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LoansAndLeasesReceivableAllowanceForLoanLossesPolicy_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Loans And Leases Receivable Allowance For Loan Losses Policy</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_IncomeRecognitionOnImpairedAndNonAccrualLoansPolicyTextBlock" xlink:type="locator" xlink:label="pbhc_IncomeRecognitionOnImpairedAndNonAccrualLoansPolicyTextBlock" />
    <link:label xml:lang="en-US" xlink:label="pbhc_IncomeRecognitionOnImpairedAndNonAccrualLoansPolicyTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Disclosure of accounting policy for income recognition on impaired and non-accrual loans.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_IncomeRecognitionOnImpairedAndNonAccrualLoansPolicyTextBlock" xlink:to="pbhc_IncomeRecognitionOnImpairedAndNonAccrualLoansPolicyTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_IncomeRecognitionOnImpairedAndNonAccrualLoansPolicyTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Income Recognition on Impaired and Nonaccrual Loans</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_IncomeRecognitionOnImpairedAndNonAccrualLoansPolicyTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Income Recognition On Impaired And Non Accrual Loans Policy [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_OffBalanceSheetCreditExposurePolicyPolicyTextBlock" xlink:type="locator" xlink:label="us-gaap_OffBalanceSheetCreditExposurePolicyPolicyTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OffBalanceSheetCreditExposurePolicyPolicyTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Off-Balance Sheet Credit Related Financial Instruments</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_OffBalanceSheetCreditExposurePolicyPolicyTextBlock" xlink:to="us-gaap_OffBalanceSheetCreditExposurePolicyPolicyTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OffBalanceSheetCreditExposurePolicyPolicyTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Off Balance Sheet Credit Exposure Policy Policy [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_PropertyPlantAndEquipmentPolicyTextBlock" xlink:type="locator" xlink:label="us-gaap_PropertyPlantAndEquipmentPolicyTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PropertyPlantAndEquipmentPolicyTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Premises and Equipment</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_PropertyPlantAndEquipmentPolicyTextBlock" xlink:to="us-gaap_PropertyPlantAndEquipmentPolicyTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PropertyPlantAndEquipmentPolicyTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Property Plant And Equipment Policy [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FinanceLoanAndLeaseReceivablesHeldForInvestmentsForeclosedAssetsPolicy" xlink:type="locator" xlink:label="us-gaap_FinanceLoanAndLeaseReceivablesHeldForInvestmentsForeclosedAssetsPolicy" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinanceLoanAndLeaseReceivablesHeldForInvestmentsForeclosedAssetsPolicy_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Foreclosed Real Estate</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FinanceLoanAndLeaseReceivablesHeldForInvestmentsForeclosedAssetsPolicy" xlink:to="us-gaap_FinanceLoanAndLeaseReceivablesHeldForInvestmentsForeclosedAssetsPolicy_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinanceLoanAndLeaseReceivablesHeldForInvestmentsForeclosedAssetsPolicy_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Finance Loan And Lease Receivables Held For Investments Foreclosed Assets Policy</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_GoodwillAndIntangibleAssetsGoodwillPolicy" xlink:type="locator" xlink:label="us-gaap_GoodwillAndIntangibleAssetsGoodwillPolicy" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_GoodwillAndIntangibleAssetsGoodwillPolicy_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Goodwill and Intangible Assets</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_GoodwillAndIntangibleAssetsGoodwillPolicy" xlink:to="us-gaap_GoodwillAndIntangibleAssetsGoodwillPolicy_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_GoodwillAndIntangibleAssetsGoodwillPolicy_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Goodwill And Intangible Assets Goodwill Policy</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_MortgageServicingRightsPolicyTextBlock" xlink:type="locator" xlink:label="pbhc_MortgageServicingRightsPolicyTextBlock" />
    <link:label xml:lang="en-US" xlink:label="pbhc_MortgageServicingRightsPolicyTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Disclosure of accounting policy for mortgage servicing rights.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_MortgageServicingRightsPolicyTextBlock" xlink:to="pbhc_MortgageServicingRightsPolicyTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_MortgageServicingRightsPolicyTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Mortgage Servicing Rights</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MortgageServicingRightsPolicyTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Mortgage Servicing Rights Policy [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ShareBasedCompensationOptionAndIncentivePlansPolicy" xlink:type="locator" xlink:label="us-gaap_ShareBasedCompensationOptionAndIncentivePlansPolicy" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationOptionAndIncentivePlansPolicy_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Stock-Based Compensation</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ShareBasedCompensationOptionAndIncentivePlansPolicy" xlink:to="us-gaap_ShareBasedCompensationOptionAndIncentivePlansPolicy_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationOptionAndIncentivePlansPolicy_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Share Based Compensation Option And Incentive Plans Policy</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_PensionAndOtherPostretirementPlansPolicy" xlink:type="locator" xlink:label="us-gaap_PensionAndOtherPostretirementPlansPolicy" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PensionAndOtherPostretirementPlansPolicy_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Retirement Benefits</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_PensionAndOtherPostretirementPlansPolicy" xlink:to="us-gaap_PensionAndOtherPostretirementPlansPolicy_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PensionAndOtherPostretirementPlansPolicy_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Pension And Other Postretirement Plans Policy</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DerivativesPolicyTextBlock" xlink:type="locator" xlink:label="us-gaap_DerivativesPolicyTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DerivativesPolicyTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Derivative Financial Instruments</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DerivativesPolicyTextBlock" xlink:to="us-gaap_DerivativesPolicyTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DerivativesPolicyTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Derivatives Policy [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_IncomeTaxPolicyTextBlock" xlink:type="locator" xlink:label="us-gaap_IncomeTaxPolicyTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncomeTaxPolicyTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Income Taxes</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_IncomeTaxPolicyTextBlock" xlink:to="us-gaap_IncomeTaxPolicyTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncomeTaxPolicyTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Income Tax Policy [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_EarningsPerSharePolicyTextBlock" xlink:type="locator" xlink:label="us-gaap_EarningsPerSharePolicyTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EarningsPerSharePolicyTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Earning Per Share</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_EarningsPerSharePolicyTextBlock" xlink:to="us-gaap_EarningsPerSharePolicyTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EarningsPerSharePolicyTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Earnings Per Share Policy [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_SegmentReportingPolicyPolicyTextBlock" xlink:type="locator" xlink:label="us-gaap_SegmentReportingPolicyPolicyTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SegmentReportingPolicyPolicyTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Segment Reporting</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_SegmentReportingPolicyPolicyTextBlock" xlink:to="us-gaap_SegmentReportingPolicyPolicyTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SegmentReportingPolicyPolicyTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Segment Reporting Policy Policy [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ComprehensiveIncomePolicyPolicyTextBlock" xlink:type="locator" xlink:label="us-gaap_ComprehensiveIncomePolicyPolicyTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ComprehensiveIncomePolicyPolicyTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Comprehensive Income (Loss)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ComprehensiveIncomePolicyPolicyTextBlock" xlink:to="us-gaap_ComprehensiveIncomePolicyPolicyTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ComprehensiveIncomePolicyPolicyTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Comprehensive Income Policy Policy [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_PriorPeriodReclassificationAdjustmentDescription" xlink:type="locator" xlink:label="us-gaap_PriorPeriodReclassificationAdjustmentDescription" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PriorPeriodReclassificationAdjustmentDescription_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Reclassifications</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_PriorPeriodReclassificationAdjustmentDescription" xlink:to="us-gaap_PriorPeriodReclassificationAdjustmentDescription_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PriorPeriodReclassificationAdjustmentDescription_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Prior Period Reclassification Adjustment Description</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ScheduleOfDerivativeInstrumentsEffectOnOtherComprehensiveIncomeLossTableTextBlock" xlink:type="locator" xlink:label="us-gaap_ScheduleOfDerivativeInstrumentsEffectOnOtherComprehensiveIncomeLossTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfDerivativeInstrumentsEffectOnOtherComprehensiveIncomeLossTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Schedule of Accumulated Other Comprehensive Loss</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ScheduleOfDerivativeInstrumentsEffectOnOtherComprehensiveIncomeLossTableTextBlock" xlink:to="us-gaap_ScheduleOfDerivativeInstrumentsEffectOnOtherComprehensiveIncomeLossTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfDerivativeInstrumentsEffectOnOtherComprehensiveIncomeLossTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Derivative Instruments Effect On Other Comprehensive Income Loss Table [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ScheduleOfEarningsPerShareBasicAndDilutedTableTextBlock" xlink:type="locator" xlink:label="us-gaap_ScheduleOfEarningsPerShareBasicAndDilutedTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfEarningsPerShareBasicAndDilutedTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Calculations of Basic and Diluted Earnings per Share</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ScheduleOfEarningsPerShareBasicAndDilutedTableTextBlock" xlink:to="us-gaap_ScheduleOfEarningsPerShareBasicAndDilutedTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfEarningsPerShareBasicAndDilutedTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Earnings Per Share Basic And Diluted Table [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_UnrealizedGainLossOnInvestmentsTableTextBlock" xlink:type="locator" xlink:label="us-gaap_UnrealizedGainLossOnInvestmentsTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_UnrealizedGainLossOnInvestmentsTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Amortized Cost and Estimated Fair Value of Investment Securities</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_UnrealizedGainLossOnInvestmentsTableTextBlock" xlink:to="us-gaap_UnrealizedGainLossOnInvestmentsTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_UnrealizedGainLossOnInvestmentsTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Unrealized Gain Loss On Investments Table [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_InvestmentsClassifiedByContractualMaturityDateTableTextBlock" xlink:type="locator" xlink:label="us-gaap_InvestmentsClassifiedByContractualMaturityDateTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InvestmentsClassifiedByContractualMaturityDateTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Amortized Cost and Estimated Fair Value of Debt Investments by Contractual Maturity</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_InvestmentsClassifiedByContractualMaturityDateTableTextBlock" xlink:to="us-gaap_InvestmentsClassifiedByContractualMaturityDateTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InvestmentsClassifiedByContractualMaturityDateTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Investments Classified By Contractual Maturity Date Table [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ScheduleOfUnrealizedLossOnInvestmentsTableTextBlock" xlink:type="locator" xlink:label="us-gaap_ScheduleOfUnrealizedLossOnInvestmentsTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfUnrealizedLossOnInvestmentsTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Investment Securities&apos; Gross Unrealized Losses and Fair Value by Investment Category and Length of Time that Individual Securities Have Continuous Unrealized Loss Position</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ScheduleOfUnrealizedLossOnInvestmentsTableTextBlock" xlink:to="us-gaap_ScheduleOfUnrealizedLossOnInvestmentsTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfUnrealizedLossOnInvestmentsTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Unrealized Loss On Investments Table [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_RealizedGainLossOnInvestmentsTableTextBlock" xlink:type="locator" xlink:label="us-gaap_RealizedGainLossOnInvestmentsTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RealizedGainLossOnInvestmentsTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Gross Realized Gains (Losses) on Sale of Securities</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_RealizedGainLossOnInvestmentsTableTextBlock" xlink:to="us-gaap_RealizedGainLossOnInvestmentsTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RealizedGainLossOnInvestmentsTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Realized Gain Loss On Investments Table [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ScheduleOfAccountsNotesLoansAndFinancingReceivableTextBlock" xlink:type="locator" xlink:label="us-gaap_ScheduleOfAccountsNotesLoansAndFinancingReceivableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfAccountsNotesLoansAndFinancingReceivableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Major Classification of Loans</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ScheduleOfAccountsNotesLoansAndFinancingReceivableTextBlock" xlink:to="us-gaap_ScheduleOfAccountsNotesLoansAndFinancingReceivableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfAccountsNotesLoansAndFinancingReceivableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Accounts Notes Loans And Financing Receivable [Text Block]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_SummaryOfPurchasedLoansPoolsTableTextBlock" xlink:type="locator" xlink:label="pbhc_SummaryOfPurchasedLoansPoolsTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="pbhc_SummaryOfPurchasedLoansPoolsTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Summary of Purchased Loans Pools Table Text Block.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_SummaryOfPurchasedLoansPoolsTableTextBlock" xlink:to="pbhc_SummaryOfPurchasedLoansPoolsTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_SummaryOfPurchasedLoansPoolsTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Summary of Purchased Loans Pools</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_SummaryOfPurchasedLoansPoolsTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Summary Of Purchased Loans Pools Table [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FinancingReceivableCreditQualityIndicatorsTableTextBlock" xlink:type="locator" xlink:label="us-gaap_FinancingReceivableCreditQualityIndicatorsTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableCreditQualityIndicatorsTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Summary of Classes of Loan Portfolio</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FinancingReceivableCreditQualityIndicatorsTableTextBlock" xlink:to="us-gaap_FinancingReceivableCreditQualityIndicatorsTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableCreditQualityIndicatorsTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Financing Receivable Credit Quality Indicators Table [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_PastDueFinancingReceivablesTableTextBlock" xlink:type="locator" xlink:label="us-gaap_PastDueFinancingReceivablesTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PastDueFinancingReceivablesTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Age Analysis of Past Due Loans Segregated by Portfolio Segment and Class of Loans</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_PastDueFinancingReceivablesTableTextBlock" xlink:to="us-gaap_PastDueFinancingReceivablesTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PastDueFinancingReceivablesTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Past Due Financing Receivables Table [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ScheduleOfFinancingReceivablesNonAccrualStatusTableTextBlock" xlink:type="locator" xlink:label="us-gaap_ScheduleOfFinancingReceivablesNonAccrualStatusTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfFinancingReceivablesNonAccrualStatusTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Nonaccrual Loans Segregated by Class of Loan</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ScheduleOfFinancingReceivablesNonAccrualStatusTableTextBlock" xlink:to="us-gaap_ScheduleOfFinancingReceivablesNonAccrualStatusTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfFinancingReceivablesNonAccrualStatusTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Financing Receivables Non Accrual Status Table [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_TroubledDebtRestructuringsOnFinancingReceivablesTableTextBlock" xlink:type="locator" xlink:label="us-gaap_TroubledDebtRestructuringsOnFinancingReceivablesTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TroubledDebtRestructuringsOnFinancingReceivablesTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Troubled Debt Restructurings on Financing Receivables</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_TroubledDebtRestructuringsOnFinancingReceivablesTableTextBlock" xlink:to="us-gaap_TroubledDebtRestructuringsOnFinancingReceivablesTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TroubledDebtRestructuringsOnFinancingReceivablesTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Troubled Debt Restructurings On Financing Receivables Table [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ImpairedFinancingReceivablesTableTextBlock" xlink:type="locator" xlink:label="us-gaap_ImpairedFinancingReceivablesTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ImpairedFinancingReceivablesTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Summary of Impaired Loans Information by Portfolio Class</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ImpairedFinancingReceivablesTableTextBlock" xlink:to="us-gaap_ImpairedFinancingReceivablesTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ImpairedFinancingReceivablesTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Impaired Financing Receivables Table [Text Block]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_AverageRecordedInvestmentInImpairedLoansTableTextBlock" xlink:type="locator" xlink:label="pbhc_AverageRecordedInvestmentInImpairedLoansTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="pbhc_AverageRecordedInvestmentInImpairedLoansTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Tabular disclosure of average recorded investment related to impaired financing receivables.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_AverageRecordedInvestmentInImpairedLoansTableTextBlock" xlink:to="pbhc_AverageRecordedInvestmentInImpairedLoansTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_AverageRecordedInvestmentInImpairedLoansTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Average Recorded Investment In Impaired Loans</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AverageRecordedInvestmentInImpairedLoansTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Average Recorded Investment In Impaired Loans Table [Text Block]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_CashBasisInterestIncomeRecognizedOnImpairedLoansTableTextBlock" xlink:type="locator" xlink:label="pbhc_CashBasisInterestIncomeRecognizedOnImpairedLoansTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="pbhc_CashBasisInterestIncomeRecognizedOnImpairedLoansTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Tabular disclosure of cash basis income recognized on impaired loan receivables.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_CashBasisInterestIncomeRecognizedOnImpairedLoansTableTextBlock" xlink:to="pbhc_CashBasisInterestIncomeRecognizedOnImpairedLoansTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_CashBasisInterestIncomeRecognizedOnImpairedLoansTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Cash Basis Interest Income Recognized On Impaired Loans</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_CashBasisInterestIncomeRecognizedOnImpairedLoansTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Cash Basis Interest Income Recognized On Impaired Loans Table [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AllowanceForCreditLossesOnFinancingReceivablesTableTextBlock" xlink:type="locator" xlink:label="us-gaap_AllowanceForCreditLossesOnFinancingReceivablesTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AllowanceForCreditLossesOnFinancingReceivablesTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Changes in the Allowance for Loan Losses</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AllowanceForCreditLossesOnFinancingReceivablesTableTextBlock" xlink:to="us-gaap_AllowanceForCreditLossesOnFinancingReceivablesTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AllowanceForCreditLossesOnFinancingReceivablesTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Allowance For Credit Losses On Financing Receivables Table [Text Block]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ScheduleOfAllowanceForLoanLossesOnBasisOfCalculationMethodologyTableTextBlock" xlink:type="locator" xlink:label="pbhc_ScheduleOfAllowanceForLoanLossesOnBasisOfCalculationMethodologyTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ScheduleOfAllowanceForLoanLossesOnBasisOfCalculationMethodologyTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Tabular disclosure of allowance for credit losses account by calculation methodology method.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ScheduleOfAllowanceForLoanLossesOnBasisOfCalculationMethodologyTableTextBlock" xlink:to="pbhc_ScheduleOfAllowanceForLoanLossesOnBasisOfCalculationMethodologyTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ScheduleOfAllowanceForLoanLossesOnBasisOfCalculationMethodologyTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Schedule of Allowance for Loan Losses on Basis of Calculation Methodology</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ScheduleOfAllowanceForLoanLossesOnBasisOfCalculationMethodologyTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Allowance For Loan Losses On Basis Of Calculation Methodology Table [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ScheduleOfServicingAssetsAtAmortizedValueTextBlock" xlink:type="locator" xlink:label="us-gaap_ScheduleOfServicingAssetsAtAmortizedValueTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfServicingAssetsAtAmortizedValueTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Mortgage Servicing Rights Capitalized and Amortized</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ScheduleOfServicingAssetsAtAmortizedValueTextBlock" xlink:to="us-gaap_ScheduleOfServicingAssetsAtAmortizedValueTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfServicingAssetsAtAmortizedValueTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Servicing Assets At Amortized Value [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_PropertyPlantAndEquipmentTextBlock" xlink:type="locator" xlink:label="us-gaap_PropertyPlantAndEquipmentTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PropertyPlantAndEquipmentTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Summary of Premises and Equipment</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_PropertyPlantAndEquipmentTextBlock" xlink:to="us-gaap_PropertyPlantAndEquipmentTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PropertyPlantAndEquipmentTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Property Plant And Equipment [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ScheduleOfFiniteLivedIntangibleAssetsTableTextBlock" xlink:type="locator" xlink:label="us-gaap_ScheduleOfFiniteLivedIntangibleAssetsTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfFiniteLivedIntangibleAssetsTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Gross Carrying Amount and Accumulated Amortization for Identifiable Intangible Asset</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ScheduleOfFiniteLivedIntangibleAssetsTableTextBlock" xlink:to="us-gaap_ScheduleOfFiniteLivedIntangibleAssetsTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfFiniteLivedIntangibleAssetsTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Finite Lived Intangible Assets Table [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ScheduleofFiniteLivedIntangibleAssetsFutureAmortizationExpenseTableTextBlock" xlink:type="locator" xlink:label="us-gaap_ScheduleofFiniteLivedIntangibleAssetsFutureAmortizationExpenseTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleofFiniteLivedIntangibleAssetsFutureAmortizationExpenseTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Estimated Amortization Expense for Each of the Five Succeeding Years</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ScheduleofFiniteLivedIntangibleAssetsFutureAmortizationExpenseTableTextBlock" xlink:to="us-gaap_ScheduleofFiniteLivedIntangibleAssetsFutureAmortizationExpenseTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleofFiniteLivedIntangibleAssetsFutureAmortizationExpenseTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Scheduleof Finite Lived Intangible Assets Future Amortization Expense Table [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DepositLiabilitiesTypeTableTextBlock" xlink:type="locator" xlink:label="us-gaap_DepositLiabilitiesTypeTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DepositLiabilitiesTypeTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Summary of Deposits</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DepositLiabilitiesTypeTableTextBlock" xlink:to="us-gaap_DepositLiabilitiesTypeTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DepositLiabilitiesTypeTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deposit Liabilities Type Table [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_TimeDepositMaturitiesTableTextBlock" xlink:type="locator" xlink:label="us-gaap_TimeDepositMaturitiesTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TimeDepositMaturitiesTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Scheduled Maturities of Time Deposits</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_TimeDepositMaturitiesTableTextBlock" xlink:to="us-gaap_TimeDepositMaturitiesTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TimeDepositMaturitiesTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Time Deposit Maturities Table [Text Block]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ScheduleOfDepositsByTypeTableTextBlock" xlink:type="locator" xlink:label="pbhc_ScheduleOfDepositsByTypeTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ScheduleOfDepositsByTypeTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Tabular disclosure of the components of deposit liabilities categorized as brokered and non-brokered deposits.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ScheduleOfDepositsByTypeTableTextBlock" xlink:to="pbhc_ScheduleOfDepositsByTypeTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ScheduleOfDepositsByTypeTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Summary of Deposits by Type</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ScheduleOfDepositsByTypeTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Deposits By Type Table [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ScheduleOfDebtTableTextBlock" xlink:type="locator" xlink:label="us-gaap_ScheduleOfDebtTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfDebtTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Composition of Borrowings</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ScheduleOfDebtTableTextBlock" xlink:to="us-gaap_ScheduleOfDebtTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfDebtTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Debt Table [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ScheduleOfMaturitiesOfLongTermDebtTableTextBlock" xlink:type="locator" xlink:label="us-gaap_ScheduleOfMaturitiesOfLongTermDebtTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfMaturitiesOfLongTermDebtTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Scheduled Maturities of Debt</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ScheduleOfMaturitiesOfLongTermDebtTableTextBlock" xlink:to="us-gaap_ScheduleOfMaturitiesOfLongTermDebtTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfMaturitiesOfLongTermDebtTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Maturities Of Long Term Debt Table [Text Block]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ComponentsOfSubordinatedBorrowingsTableTextBlock" xlink:type="locator" xlink:label="pbhc_ComponentsOfSubordinatedBorrowingsTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ComponentsOfSubordinatedBorrowingsTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Tabular disclosure of information pertaining to Subordinated debt instruments or arrangements, including but not limited to identification of terms, features, collateral requirements and other information necessary to a fair presentation.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ComponentsOfSubordinatedBorrowingsTableTextBlock" xlink:to="pbhc_ComponentsOfSubordinatedBorrowingsTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ComponentsOfSubordinatedBorrowingsTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Composition of Subordinated Loans</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ComponentsOfSubordinatedBorrowingsTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Components Of Subordinated Borrowings Table [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ScheduleOfSubordinatedBorrowingTextBlock" xlink:type="locator" xlink:label="us-gaap_ScheduleOfSubordinatedBorrowingTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfSubordinatedBorrowingTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Schedule of Principal Balances, Interest Rates and Maturities of the Subordinated Loans</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ScheduleOfSubordinatedBorrowingTextBlock" xlink:to="us-gaap_ScheduleOfSubordinatedBorrowingTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfSubordinatedBorrowingTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Subordinated Borrowing [Text Block]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ScheduleOfRepaymentsOfSubordinatedBorrowingsTableTextBlock" xlink:type="locator" xlink:label="pbhc_ScheduleOfRepaymentsOfSubordinatedBorrowingsTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ScheduleOfRepaymentsOfSubordinatedBorrowingsTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Tabular disclosure of repayment of subordinated borrowing where a lender is placed in a lien position behind debt having a higher priority of repayment (senior) in case of liquidation of the entity&apos;s assets or underlying collateral.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ScheduleOfRepaymentsOfSubordinatedBorrowingsTableTextBlock" xlink:to="pbhc_ScheduleOfRepaymentsOfSubordinatedBorrowingsTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ScheduleOfRepaymentsOfSubordinatedBorrowingsTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Scheduled Repayments of the Subordinated Loans</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ScheduleOfRepaymentsOfSubordinatedBorrowingsTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Repayments Of Subordinated Borrowings Table [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ScheduleOfAccumulatedBenefitObligationsInExcessOfFairValueOfPlanAssetsTableTextBlock" xlink:type="locator" xlink:label="us-gaap_ScheduleOfAccumulatedBenefitObligationsInExcessOfFairValueOfPlanAssetsTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfAccumulatedBenefitObligationsInExcessOfFairValueOfPlanAssetsTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Changes in Plan Benefit Obligations, Fair Value of Plan Assets and Plans&apos; Funded Status</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ScheduleOfAccumulatedBenefitObligationsInExcessOfFairValueOfPlanAssetsTableTextBlock" xlink:to="us-gaap_ScheduleOfAccumulatedBenefitObligationsInExcessOfFairValueOfPlanAssetsTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfAccumulatedBenefitObligationsInExcessOfFairValueOfPlanAssetsTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Accumulated Benefit Obligations In Excess Of Fair Value Of Plan Assets Table [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ScheduleOfAmountsRecognizedInOtherComprehensiveIncomeLossTableTextBlock" xlink:type="locator" xlink:label="us-gaap_ScheduleOfAmountsRecognizedInOtherComprehensiveIncomeLossTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfAmountsRecognizedInOtherComprehensiveIncomeLossTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Amounts Recognized in Other Comprehensive Loss</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ScheduleOfAmountsRecognizedInOtherComprehensiveIncomeLossTableTextBlock" xlink:to="us-gaap_ScheduleOfAmountsRecognizedInOtherComprehensiveIncomeLossTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfAmountsRecognizedInOtherComprehensiveIncomeLossTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Amounts Recognized In Other Comprehensive Income Loss Table [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ScheduleOfAssumptionsUsedTableTextBlock" xlink:type="locator" xlink:label="us-gaap_ScheduleOfAssumptionsUsedTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfAssumptionsUsedTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Significant Assumptions Used in Determining Benefit Obligations and Net Periodic Benefit Plan Cost</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ScheduleOfAssumptionsUsedTableTextBlock" xlink:to="us-gaap_ScheduleOfAssumptionsUsedTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfAssumptionsUsedTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Assumptions Used Table [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ScheduleOfNetBenefitCostsTableTextBlock" xlink:type="locator" xlink:label="us-gaap_ScheduleOfNetBenefitCostsTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfNetBenefitCostsTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Composition of Net Periodic Benefit Plan (Benefit) Cost</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ScheduleOfNetBenefitCostsTableTextBlock" xlink:to="us-gaap_ScheduleOfNetBenefitCostsTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfNetBenefitCostsTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Net Benefit Costs Table [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ScheduleOfDefinedBenefitPlansDisclosuresTextBlock" xlink:type="locator" xlink:label="us-gaap_ScheduleOfDefinedBenefitPlansDisclosuresTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfDefinedBenefitPlansDisclosuresTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Pension Plan Assets Measured at Fair Value</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ScheduleOfDefinedBenefitPlansDisclosuresTextBlock" xlink:to="us-gaap_ScheduleOfDefinedBenefitPlansDisclosuresTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfDefinedBenefitPlansDisclosuresTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Defined Benefit Plans Disclosures [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ScheduleOfExpectedBenefitPaymentsTableTextBlock" xlink:type="locator" xlink:label="us-gaap_ScheduleOfExpectedBenefitPaymentsTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfExpectedBenefitPaymentsTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Expected Future Service Benefit Payments</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ScheduleOfExpectedBenefitPaymentsTableTextBlock" xlink:to="us-gaap_ScheduleOfExpectedBenefitPaymentsTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfExpectedBenefitPaymentsTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Expected Benefit Payments Table [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ScheduleOfShareBasedCompensationStockOptionsActivityTableTextBlock" xlink:type="locator" xlink:label="us-gaap_ScheduleOfShareBasedCompensationStockOptionsActivityTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfShareBasedCompensationStockOptionsActivityTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Activity in the Stock Option Plans</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ScheduleOfShareBasedCompensationStockOptionsActivityTableTextBlock" xlink:to="us-gaap_ScheduleOfShareBasedCompensationStockOptionsActivityTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfShareBasedCompensationStockOptionsActivityTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Share Based Compensation Stock Options Activity Table [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ScheduleOfComponentsOfIncomeTaxExpenseBenefitTableTextBlock" xlink:type="locator" xlink:label="us-gaap_ScheduleOfComponentsOfIncomeTaxExpenseBenefitTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfComponentsOfIncomeTaxExpenseBenefitTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Provision for Income Taxes</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ScheduleOfComponentsOfIncomeTaxExpenseBenefitTableTextBlock" xlink:to="us-gaap_ScheduleOfComponentsOfIncomeTaxExpenseBenefitTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfComponentsOfIncomeTaxExpenseBenefitTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Components Of Income Tax Expense Benefit Table [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ScheduleOfDeferredTaxAssetsAndLiabilitiesTableTextBlock" xlink:type="locator" xlink:label="us-gaap_ScheduleOfDeferredTaxAssetsAndLiabilitiesTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfDeferredTaxAssetsAndLiabilitiesTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Components of the Net Deferred Tax Asset (Liability) Included in Other Assets</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ScheduleOfDeferredTaxAssetsAndLiabilitiesTableTextBlock" xlink:to="us-gaap_ScheduleOfDeferredTaxAssetsAndLiabilitiesTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfDeferredTaxAssetsAndLiabilitiesTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Deferred Tax Assets And Liabilities Table [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ScheduleOfEffectiveIncomeTaxRateReconciliationTableTextBlock" xlink:type="locator" xlink:label="us-gaap_ScheduleOfEffectiveIncomeTaxRateReconciliationTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfEffectiveIncomeTaxRateReconciliationTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Reconciliation of the Federal Statutory Income Tax Rate to the Effective Income Tax Rate</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ScheduleOfEffectiveIncomeTaxRateReconciliationTableTextBlock" xlink:to="us-gaap_ScheduleOfEffectiveIncomeTaxRateReconciliationTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfEffectiveIncomeTaxRateReconciliationTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Effective Income Tax Rate Reconciliation Table [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ScheduleOfFairValueOffBalanceSheetRisksTextBlock" xlink:type="locator" xlink:label="us-gaap_ScheduleOfFairValueOffBalanceSheetRisksTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfFairValueOffBalanceSheetRisksTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Summary of the Contractual Amounts of Financial Instruments with Credit Risk</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ScheduleOfFairValueOffBalanceSheetRisksTextBlock" xlink:to="us-gaap_ScheduleOfFairValueOffBalanceSheetRisksTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfFairValueOffBalanceSheetRisksTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Fair Value Off Balance Sheet Risks [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ScheduleOfComplianceWithRegulatoryCapitalRequirementsUnderBankingRegulationsTextBlock" xlink:type="locator" xlink:label="us-gaap_ScheduleOfComplianceWithRegulatoryCapitalRequirementsUnderBankingRegulationsTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfComplianceWithRegulatoryCapitalRequirementsUnderBankingRegulationsTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Actual Capital Amounts and Ratios</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ScheduleOfComplianceWithRegulatoryCapitalRequirementsUnderBankingRegulationsTextBlock" xlink:to="us-gaap_ScheduleOfComplianceWithRegulatoryCapitalRequirementsUnderBankingRegulationsTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfComplianceWithRegulatoryCapitalRequirementsUnderBankingRegulationsTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Compliance With Regulatory Capital Requirements Under Banking Regulations [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ScheduleOfFairValueHedgingInstrumentsStatementsOfFinancialPerformanceAndFinancialPositionLocationTableTextBlock" xlink:type="locator" xlink:label="us-gaap_ScheduleOfFairValueHedgingInstrumentsStatementsOfFinancialPerformanceAndFinancialPositionLocationTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfFairValueHedgingInstrumentsStatementsOfFinancialPerformanceAndFinancialPositionLocationTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Cumulative Basis Adjustments for Fair Value Hedges</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ScheduleOfFairValueHedgingInstrumentsStatementsOfFinancialPerformanceAndFinancialPositionLocationTableTextBlock" xlink:to="us-gaap_ScheduleOfFairValueHedgingInstrumentsStatementsOfFinancialPerformanceAndFinancialPositionLocationTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfFairValueHedgingInstrumentsStatementsOfFinancialPerformanceAndFinancialPositionLocationTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Fair Value Hedging Instruments Statements Of Financial Performance And Financial Position Location Table [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ScheduleOfCashFlowHedgingInstrumentsStatementsOfFinancialPerformanceAndFinancialPositionLocationTableTextBlock" xlink:type="locator" xlink:label="us-gaap_ScheduleOfCashFlowHedgingInstrumentsStatementsOfFinancialPerformanceAndFinancialPositionLocationTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfCashFlowHedgingInstrumentsStatementsOfFinancialPerformanceAndFinancialPositionLocationTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Schedule of Cash Flow Hedges</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ScheduleOfCashFlowHedgingInstrumentsStatementsOfFinancialPerformanceAndFinancialPositionLocationTableTextBlock" xlink:to="us-gaap_ScheduleOfCashFlowHedgingInstrumentsStatementsOfFinancialPerformanceAndFinancialPositionLocationTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfCashFlowHedgingInstrumentsStatementsOfFinancialPerformanceAndFinancialPositionLocationTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Cash Flow Hedging Instruments Statements Of Financial Performance And Financial Position Location Table [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FairValueAssetsMeasuredOnRecurringBasisTextBlock" xlink:type="locator" xlink:label="us-gaap_FairValueAssetsMeasuredOnRecurringBasisTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueAssetsMeasuredOnRecurringBasisTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Fair Value of Assets on Recurring Basis Segregated by Level of Valuation Inputs</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FairValueAssetsMeasuredOnRecurringBasisTextBlock" xlink:to="us-gaap_FairValueAssetsMeasuredOnRecurringBasisTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueAssetsMeasuredOnRecurringBasisTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Fair Value Assets Measured On Recurring Basis [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FairValueAssetsMeasuredOnNonrecurringBasisTextBlock" xlink:type="locator" xlink:label="us-gaap_FairValueAssetsMeasuredOnNonrecurringBasisTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueAssetsMeasuredOnNonrecurringBasisTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Summary of Fair Value Assets Measured on Nonrecurring Basis</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FairValueAssetsMeasuredOnNonrecurringBasisTextBlock" xlink:to="us-gaap_FairValueAssetsMeasuredOnNonrecurringBasisTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueAssetsMeasuredOnNonrecurringBasisTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Fair Value Assets Measured On Nonrecurring Basis [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisValuationTechniquesTableTextBlock" xlink:type="locator" xlink:label="us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisValuationTechniquesTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisValuationTechniquesTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Fair Value Inputs, Quantitative Information</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisValuationTechniquesTableTextBlock" xlink:to="us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisValuationTechniquesTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisValuationTechniquesTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Fair Value Assets And Liabilities Measured On Recurring And Nonrecurring Basis Valuation Techniques Table [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FairValueByBalanceSheetGroupingTextBlock" xlink:type="locator" xlink:label="us-gaap_FairValueByBalanceSheetGroupingTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueByBalanceSheetGroupingTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Carrying Amounts and Fair Value of Financial Instruments</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FairValueByBalanceSheetGroupingTextBlock" xlink:to="us-gaap_FairValueByBalanceSheetGroupingTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueByBalanceSheetGroupingTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Fair Value By Balance Sheet Grouping [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/srt/2020/elts/srt-2020-01-31.xsd#srt_ScheduleOfCondensedFinancialStatementsTableTextBlock" xlink:type="locator" xlink:label="srt_ScheduleOfCondensedFinancialStatementsTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="srt_ScheduleOfCondensedFinancialStatementsTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Schedule of Condensed Financial Information</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="srt_ScheduleOfCondensedFinancialStatementsTableTextBlock" xlink:to="srt_ScheduleOfCondensedFinancialStatementsTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="srt_ScheduleOfCondensedFinancialStatementsTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Condensed Financial Statements Table [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ScheduleOfRelatedPartyTransactionsTableTextBlock" xlink:type="locator" xlink:label="us-gaap_ScheduleOfRelatedPartyTransactionsTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfRelatedPartyTransactionsTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Schedule of Loans to Related Parties</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ScheduleOfRelatedPartyTransactionsTableTextBlock" xlink:to="us-gaap_ScheduleOfRelatedPartyTransactionsTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfRelatedPartyTransactionsTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Related Party Transactions Table [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ScheduleOfAccumulatedOtherComprehensiveIncomeLossTableTextBlock" xlink:type="locator" xlink:label="us-gaap_ScheduleOfAccumulatedOtherComprehensiveIncomeLossTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfAccumulatedOtherComprehensiveIncomeLossTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Changes in the Components of Accumulated Other Comprehensive Income (Loss), Net of Tax</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ScheduleOfAccumulatedOtherComprehensiveIncomeLossTableTextBlock" xlink:to="us-gaap_ScheduleOfAccumulatedOtherComprehensiveIncomeLossTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfAccumulatedOtherComprehensiveIncomeLossTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Accumulated Other Comprehensive Income Loss Table [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ReclassificationOutOfAccumulatedOtherComprehensiveIncomeTableTextBlock" xlink:type="locator" xlink:label="us-gaap_ReclassificationOutOfAccumulatedOtherComprehensiveIncomeTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ReclassificationOutOfAccumulatedOtherComprehensiveIncomeTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Schedule of Amounts Reclassified Out of Each Component of AOCI</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ReclassificationOutOfAccumulatedOtherComprehensiveIncomeTableTextBlock" xlink:to="us-gaap_ReclassificationOutOfAccumulatedOtherComprehensiveIncomeTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ReclassificationOutOfAccumulatedOtherComprehensiveIncomeTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Reclassification Out Of Accumulated Other Comprehensive Income Table [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DisaggregationOfRevenueTableTextBlock" xlink:type="locator" xlink:label="us-gaap_DisaggregationOfRevenueTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DisaggregationOfRevenueTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Summary of Noninterest Income</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DisaggregationOfRevenueTableTextBlock" xlink:to="us-gaap_DisaggregationOfRevenueTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DisaggregationOfRevenueTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Disaggregation Of Revenue Table [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LeaseCostTableTextBlock" xlink:type="locator" xlink:label="us-gaap_LeaseCostTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LeaseCostTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Components of Lease Expense</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LeaseCostTableTextBlock" xlink:to="us-gaap_LeaseCostTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LeaseCostTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Lease Cost Table [Text Block]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ScheduleOfSupplementalCashFlowInformationRelatedToLeasesTableTextBlock" xlink:type="locator" xlink:label="pbhc_ScheduleOfSupplementalCashFlowInformationRelatedToLeasesTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ScheduleOfSupplementalCashFlowInformationRelatedToLeasesTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Schedule of supplemental cash flow information related to leases.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ScheduleOfSupplementalCashFlowInformationRelatedToLeasesTableTextBlock" xlink:to="pbhc_ScheduleOfSupplementalCashFlowInformationRelatedToLeasesTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ScheduleOfSupplementalCashFlowInformationRelatedToLeasesTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Supplemental Cash Flow Information Related to Leases</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ScheduleOfSupplementalCashFlowInformationRelatedToLeasesTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Supplemental Cash Flow Information Related To Leases Table [Text Block]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ScheduleOfSupplementalBalanceSheetInformationRelatedToLeasesTableTextBlock" xlink:type="locator" xlink:label="pbhc_ScheduleOfSupplementalBalanceSheetInformationRelatedToLeasesTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ScheduleOfSupplementalBalanceSheetInformationRelatedToLeasesTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Schedule of supplemental balance sheet information related to leases.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ScheduleOfSupplementalBalanceSheetInformationRelatedToLeasesTableTextBlock" xlink:to="pbhc_ScheduleOfSupplementalBalanceSheetInformationRelatedToLeasesTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ScheduleOfSupplementalBalanceSheetInformationRelatedToLeasesTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Supplemental Balance Sheet Information Related to Leases</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ScheduleOfSupplementalBalanceSheetInformationRelatedToLeasesTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Supplemental Balance Sheet Information Related To Leases Table [Text Block]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LesseeOperatingLeaseLiabilityMaturityTableTextBlock" xlink:type="locator" xlink:label="us-gaap_LesseeOperatingLeaseLiabilityMaturityTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LesseeOperatingLeaseLiabilityMaturityTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Maturities of Lease Liabilities</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LesseeOperatingLeaseLiabilityMaturityTableTextBlock" xlink:to="us-gaap_LesseeOperatingLeaseLiabilityMaturityTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LesseeOperatingLeaseLiabilityMaturityTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Lessee Operating Lease Liability Maturity Table [Text Block]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ScheduleOfLoanPortfolioByCollateralTypeWithinMajorCategoriesTableTextBlock" xlink:type="locator" xlink:label="pbhc_ScheduleOfLoanPortfolioByCollateralTypeWithinMajorCategoriesTableTextBlock" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ScheduleOfLoanPortfolioByCollateralTypeWithinMajorCategoriesTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Schedule of loan portfolio by collateral type within major categories.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ScheduleOfLoanPortfolioByCollateralTypeWithinMajorCategoriesTableTextBlock" xlink:to="pbhc_ScheduleOfLoanPortfolioByCollateralTypeWithinMajorCategoriesTableTextBlock_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ScheduleOfLoanPortfolioByCollateralTypeWithinMajorCategoriesTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Summary of Loan Portfolio by Collateral Type Within Major Categories</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ScheduleOfLoanPortfolioByCollateralTypeWithinMajorCategoriesTableTextBlock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Loan Portfolio By Collateral Type Within Major Categories Table [Text Block]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_NumberOfWhollyOwnedSubsidiaries" xlink:type="locator" xlink:label="pbhc_NumberOfWhollyOwnedSubsidiaries" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfWhollyOwnedSubsidiaries_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The number of wholly-owned subsidiaries.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_NumberOfWhollyOwnedSubsidiaries" xlink:to="pbhc_NumberOfWhollyOwnedSubsidiaries_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ConsolidationOfMembershipInterestInSubsidiary" xlink:type="locator" xlink:label="pbhc_ConsolidationOfMembershipInterestInSubsidiary" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ConsolidationOfMembershipInterestInSubsidiary_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Refers to percentage of consolidation of interest own in Fitzgibbons.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ConsolidationOfMembershipInterestInSubsidiary" xlink:to="pbhc_ConsolidationOfMembershipInterestInSubsidiary_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_NatureOfOperationsAbstract" xlink:type="locator" xlink:label="pbhc_NatureOfOperationsAbstract" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NatureOfOperationsAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Nature of Operations [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_NatureOfOperationsAbstract" xlink:to="pbhc_NatureOfOperationsAbstract_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_NetProceedsFromIssuanceOfPrivatePlacement" xlink:type="locator" xlink:label="pbhc_NetProceedsFromIssuanceOfPrivatePlacement" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NetProceedsFromIssuanceOfPrivatePlacement_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Net proceeds from issuance of private placement.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_NetProceedsFromIssuanceOfPrivatePlacement" xlink:to="pbhc_NetProceedsFromIssuanceOfPrivatePlacement_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_NatureOfOperationsLineItems" xlink:type="locator" xlink:label="pbhc_NatureOfOperationsLineItems" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NatureOfOperationsLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Line items represent financial concepts included in a table. These concepts are used to disclose reportable information associated with domain members defined in one or many axes to the table.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_NatureOfOperationsLineItems" xlink:to="pbhc_NatureOfOperationsLineItems_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_NatureOfOperationsTable" xlink:type="locator" xlink:label="pbhc_NatureOfOperationsTable" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NatureOfOperationsTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Disclosure of information about nature of operations related to entity.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_NatureOfOperationsTable" xlink:to="pbhc_NatureOfOperationsTable_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NatureOfOperationsTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Nature Of Operations [Table]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NatureOfOperationsTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Nature Of Operations [Table]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_FitzGibbonsAgencyLLCMember" xlink:type="locator" xlink:label="pbhc_FitzGibbonsAgencyLLCMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_FitzGibbonsAgencyLLCMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">FitzGibbons agency LLC.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_FitzGibbonsAgencyLLCMember" xlink:to="pbhc_FitzGibbonsAgencyLLCMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_FitzGibbonsAgencyLLCMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">FitzGibbons Agency LLC [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_FitzGibbonsAgencyLLCMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Fitz Gibbons Agency L L C [Member]</link:label>
    <link:loc xlink:href="https://xbrl.sec.gov/dei/2020/dei-2020-01-31.xsd#dei_LegalEntityAxis" xlink:type="locator" xlink:label="dei_LegalEntityAxis" />
    <link:label xml:lang="en-US" xlink:label="dei_LegalEntityAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Legal Entity</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="dei_LegalEntityAxis" xlink:to="dei_LegalEntityAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="dei_LegalEntityAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Legal Entity [Axis]</link:label>
    <link:loc xlink:href="https://xbrl.sec.gov/dei/2020/dei-2020-01-31.xsd#dei_EntityDomain" xlink:type="locator" xlink:label="dei_EntityDomain" />
    <link:label xml:lang="en-US" xlink:label="dei_EntityDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Entity</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="dei_EntityDomain" xlink:to="dei_EntityDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="dei_EntityDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Entity [Domain]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_PathfinderRiskManagementCompanyIncMember" xlink:type="locator" xlink:label="pbhc_PathfinderRiskManagementCompanyIncMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_PathfinderRiskManagementCompanyIncMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Pathfinder risk management company Inc.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_PathfinderRiskManagementCompanyIncMember" xlink:to="pbhc_PathfinderRiskManagementCompanyIncMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_PathfinderRiskManagementCompanyIncMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Pathfinder Risk Management Company Inc [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_PathfinderRiskManagementCompanyIncMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Pathfinder Risk Management Company Inc [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/srt/2020/elts/srt-2020-01-31.xsd#srt_ConsolidatedEntitiesAxis" xlink:type="locator" xlink:label="srt_ConsolidatedEntitiesAxis" />
    <link:label xml:lang="en-US" xlink:label="srt_ConsolidatedEntitiesAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Consolidated Entities</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="srt_ConsolidatedEntitiesAxis" xlink:to="srt_ConsolidatedEntitiesAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="srt_ConsolidatedEntitiesAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Consolidated Entities [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/srt/2020/elts/srt-2020-01-31.xsd#srt_ConsolidatedEntitiesDomain" xlink:type="locator" xlink:label="srt_ConsolidatedEntitiesDomain" />
    <link:label xml:lang="en-US" xlink:label="srt_ConsolidatedEntitiesDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Consolidated Entities</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="srt_ConsolidatedEntitiesDomain" xlink:to="srt_ConsolidatedEntitiesDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="srt_ConsolidatedEntitiesDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Consolidated Entities [Domain]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/srt/2020/elts/srt-2020-01-31.xsd#srt_ParentCompanyMember" xlink:type="locator" xlink:label="srt_ParentCompanyMember" />
    <link:label xml:lang="en-US" xlink:label="srt_ParentCompanyMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Pathfinder Bank [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="srt_ParentCompanyMember" xlink:to="srt_ParentCompanyMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="srt_ParentCompanyMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Parent Company [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LongtermDebtTypeAxis" xlink:type="locator" xlink:label="us-gaap_LongtermDebtTypeAxis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LongtermDebtTypeAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Long-term Debt, Type</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LongtermDebtTypeAxis" xlink:to="us-gaap_LongtermDebtTypeAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LongtermDebtTypeAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Longterm Debt Type [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LongtermDebtTypeDomain" xlink:type="locator" xlink:label="us-gaap_LongtermDebtTypeDomain" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LongtermDebtTypeDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Long-term Debt, Type</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LongtermDebtTypeDomain" xlink:to="us-gaap_LongtermDebtTypeDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LongtermDebtTypeDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Longterm Debt Type [Domain]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_SubordinatedDebtMember" xlink:type="locator" xlink:label="us-gaap_SubordinatedDebtMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SubordinatedDebtMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Subordinated Debt [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_SubordinatedDebtMember" xlink:to="us-gaap_SubordinatedDebtMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SubordinatedDebtMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Subordinated Debt [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_SBLFPreferredStockMember" xlink:type="locator" xlink:label="pbhc_SBLFPreferredStockMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_SBLFPreferredStockMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">SBLF Preferred Stock.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_SBLFPreferredStockMember" xlink:to="pbhc_SBLFPreferredStockMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_SBLFPreferredStockMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">SBLF Preferred stock [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_SBLFPreferredStockMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">S B L F Preferred Stock [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/srt/2020/elts/srt-2020-01-31.xsd#srt_RangeAxis" xlink:type="locator" xlink:label="srt_RangeAxis" />
    <link:label xml:lang="en-US" xlink:label="srt_RangeAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Statistical Measurement</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="srt_RangeAxis" xlink:to="srt_RangeAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="srt_RangeAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Range [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/srt/2020/elts/srt-2020-01-31.xsd#srt_RangeMember" xlink:type="locator" xlink:label="srt_RangeMember" />
    <link:label xml:lang="en-US" xlink:label="srt_RangeMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Statistical Measurement</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="srt_RangeMember" xlink:to="srt_RangeMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="srt_RangeMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Range [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/srt/2020/elts/srt-2020-01-31.xsd#srt_MaximumMember" xlink:type="locator" xlink:label="srt_MaximumMember" />
    <link:label xml:lang="en-US" xlink:label="srt_MaximumMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Maximum [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="srt_MaximumMember" xlink:to="srt_MaximumMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="srt_MaximumMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Maximum [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/srt/2020/elts/srt-2020-01-31.xsd#srt_MinimumMember" xlink:type="locator" xlink:label="srt_MinimumMember" />
    <link:label xml:lang="en-US" xlink:label="srt_MinimumMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Minimum [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="srt_MinimumMember" xlink:to="srt_MinimumMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="srt_MinimumMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Minimum [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_PropertyPlantAndEquipmentByTypeAxis" xlink:type="locator" xlink:label="us-gaap_PropertyPlantAndEquipmentByTypeAxis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PropertyPlantAndEquipmentByTypeAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Long-Lived Tangible Asset</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_PropertyPlantAndEquipmentByTypeAxis" xlink:to="us-gaap_PropertyPlantAndEquipmentByTypeAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PropertyPlantAndEquipmentByTypeAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Property Plant And Equipment By Type [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_PropertyPlantAndEquipmentTypeDomain" xlink:type="locator" xlink:label="us-gaap_PropertyPlantAndEquipmentTypeDomain" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PropertyPlantAndEquipmentTypeDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Long-Lived Tangible Asset</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_PropertyPlantAndEquipmentTypeDomain" xlink:to="us-gaap_PropertyPlantAndEquipmentTypeDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PropertyPlantAndEquipmentTypeDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Property Plant And Equipment Type [Domain]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_BuildingAndBuildingImprovementsMember" xlink:type="locator" xlink:label="us-gaap_BuildingAndBuildingImprovementsMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_BuildingAndBuildingImprovementsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Premises [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_BuildingAndBuildingImprovementsMember" xlink:to="us-gaap_BuildingAndBuildingImprovementsMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_BuildingAndBuildingImprovementsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Building And Building Improvements [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_EquipmentMember" xlink:type="locator" xlink:label="us-gaap_EquipmentMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EquipmentMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Equipment [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_EquipmentMember" xlink:to="us-gaap_EquipmentMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EquipmentMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Equipment [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DerivativeInstrumentRiskAxis" xlink:type="locator" xlink:label="us-gaap_DerivativeInstrumentRiskAxis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DerivativeInstrumentRiskAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Derivative Instrument</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DerivativeInstrumentRiskAxis" xlink:to="us-gaap_DerivativeInstrumentRiskAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DerivativeInstrumentRiskAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Derivative Instrument Risk [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DerivativeContractTypeDomain" xlink:type="locator" xlink:label="us-gaap_DerivativeContractTypeDomain" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DerivativeContractTypeDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Derivative Contract</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DerivativeContractTypeDomain" xlink:to="us-gaap_DerivativeContractTypeDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DerivativeContractTypeDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Derivative Contract Type [Domain]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_InterestRateContractMember" xlink:type="locator" xlink:label="us-gaap_InterestRateContractMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestRateContractMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Interest Rate Derivatives [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_InterestRateContractMember" xlink:to="us-gaap_InterestRateContractMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestRateContractMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Interest Rate Contract [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NatureOfOperationsLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Nature Of Operations [Line Items]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NatureOfOperationsLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Nature Of Operations [Line Items]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NatureOfOperationsAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Nature of Operations [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NatureOfOperationsAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Nature Of Operations [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ProceedsFromIssuanceOfCommonStock" xlink:type="locator" xlink:label="us-gaap_ProceedsFromIssuanceOfCommonStock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ProceedsFromIssuanceOfCommonStock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Amount received of net proceeds</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ProceedsFromIssuanceOfCommonStock" xlink:to="us-gaap_ProceedsFromIssuanceOfCommonStock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ProceedsFromIssuanceOfCommonStock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Proceeds From Issuance Of Common Stock</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_StockIssuedDuringPeriodSharesNewIssues" xlink:type="locator" xlink:label="us-gaap_StockIssuedDuringPeriodSharesNewIssues" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_StockIssuedDuringPeriodSharesNewIssues_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Number of shares of common stock sold (in shares)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_StockIssuedDuringPeriodSharesNewIssues" xlink:to="us-gaap_StockIssuedDuringPeriodSharesNewIssues_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_StockIssuedDuringPeriodSharesNewIssues_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Stock Issued During Period Shares New Issues</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DebtInstrumentFaceAmount" xlink:type="locator" xlink:label="us-gaap_DebtInstrumentFaceAmount" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtInstrumentFaceAmount_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Subordinated loan face value</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DebtInstrumentFaceAmount" xlink:to="us-gaap_DebtInstrumentFaceAmount_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtInstrumentFaceAmount_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Debt Instrument Face Amount</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_PaymentsForRepurchaseOfPreferredStockAndPreferenceStock" xlink:type="locator" xlink:label="us-gaap_PaymentsForRepurchaseOfPreferredStockAndPreferenceStock" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PaymentsForRepurchaseOfPreferredStockAndPreferenceStock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Retirement of preferred stock</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_PaymentsForRepurchaseOfPreferredStockAndPreferenceStock" xlink:to="us-gaap_PaymentsForRepurchaseOfPreferredStockAndPreferenceStock_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PaymentsForRepurchaseOfPreferredStockAndPreferenceStock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Payments For Repurchase Of Preferred Stock And Preference Stock</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NetProceedsFromIssuanceOfPrivatePlacement_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Net proceeds from private placement</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NetProceedsFromIssuanceOfPrivatePlacement_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Net Proceeds From Issuance Of Private Placement</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfWhollyOwnedSubsidiaries_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Number of wholly-owned subsidiaries</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfWhollyOwnedSubsidiaries_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Number Of Wholly Owned Subsidiaries</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_MinorityInterestOwnershipPercentageByParent" xlink:type="locator" xlink:label="us-gaap_MinorityInterestOwnershipPercentageByParent" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_MinorityInterestOwnershipPercentageByParent_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Membership interest own in Fitzgibbons through subsidiary</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_MinorityInterestOwnershipPercentageByParent" xlink:to="us-gaap_MinorityInterestOwnershipPercentageByParent_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_MinorityInterestOwnershipPercentageByParent_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Minority Interest Ownership Percentage By Parent</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ConsolidationOfMembershipInterestInSubsidiary_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Consolidation of membership interest in Fitzgibbons</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ConsolidationOfMembershipInterestInSubsidiary_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Consolidation Of Membership Interest In Subsidiary</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_MinorityInterestOwnershipPercentageByNoncontrollingOwners" xlink:type="locator" xlink:label="us-gaap_MinorityInterestOwnershipPercentageByNoncontrollingOwners" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_MinorityInterestOwnershipPercentageByNoncontrollingOwners_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Noncontrolling interest by subsidiary</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_MinorityInterestOwnershipPercentageByNoncontrollingOwners" xlink:to="us-gaap_MinorityInterestOwnershipPercentageByNoncontrollingOwners_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_MinorityInterestOwnershipPercentageByNoncontrollingOwners_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Minority Interest Ownership Percentage By Noncontrolling Owners</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_NumberOfOfficesLocatedInOswegoCounty" xlink:type="locator" xlink:label="pbhc_NumberOfOfficesLocatedInOswegoCounty" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfOfficesLocatedInOswegoCounty_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The number of offices located in Oswego county.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_NumberOfOfficesLocatedInOswegoCounty" xlink:to="pbhc_NumberOfOfficesLocatedInOswegoCounty_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_NumberOfOfficesLocatedInOnondagaCounty" xlink:type="locator" xlink:label="pbhc_NumberOfOfficesLocatedInOnondagaCounty" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfOfficesLocatedInOnondagaCounty_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The number of offices located in Onondaga county.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_NumberOfOfficesLocatedInOnondagaCounty" xlink:to="pbhc_NumberOfOfficesLocatedInOnondagaCounty_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_NumberOfLoanProductionOfficeLocatedInOneidaCounty" xlink:type="locator" xlink:label="pbhc_NumberOfLoanProductionOfficeLocatedInOneidaCounty" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfLoanProductionOfficeLocatedInOneidaCounty_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The number of loan production office located in Oneida county.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_NumberOfLoanProductionOfficeLocatedInOneidaCounty" xlink:to="pbhc_NumberOfLoanProductionOfficeLocatedInOneidaCounty_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfOfficesLocatedInOswegoCounty_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Number of offices located In Oswego County</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfOfficesLocatedInOswegoCounty_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Number Of Offices Located In Oswego County</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfOfficesLocatedInOnondagaCounty_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Number of offices located in Onondaga County</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfOfficesLocatedInOnondagaCounty_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Number Of Offices Located In Onondaga County</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfLoanProductionOfficeLocatedInOneidaCounty_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Number of loan production office located in Oneida County</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfLoanProductionOfficeLocatedInOneidaCounty_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Number Of Loan Production Office Located In Oneida County</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_MinimumCommercialRealEstateLoansAmountRequiredForAdditionalReview" xlink:type="locator" xlink:label="pbhc_MinimumCommercialRealEstateLoansAmountRequiredForAdditionalReview" />
    <link:label xml:lang="en-US" xlink:label="pbhc_MinimumCommercialRealEstateLoansAmountRequiredForAdditionalReview_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Minimum amount of commercial real estate loans which requires additional reviews by a qualified third party.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_MinimumCommercialRealEstateLoansAmountRequiredForAdditionalReview" xlink:to="pbhc_MinimumCommercialRealEstateLoansAmountRequiredForAdditionalReview_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_SignificantGroupConcentrationsOfCreditRiskAbstract" xlink:type="locator" xlink:label="pbhc_SignificantGroupConcentrationsOfCreditRiskAbstract" />
    <link:label xml:lang="en-US" xlink:label="pbhc_SignificantGroupConcentrationsOfCreditRiskAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Significant Group Concentrations of Credit Risk [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_SignificantGroupConcentrationsOfCreditRiskAbstract" xlink:to="pbhc_SignificantGroupConcentrationsOfCreditRiskAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_SignificantGroupConcentrationsOfCreditRiskAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Significant Group Concentrations of Credit Risk [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_SignificantGroupConcentrationsOfCreditRiskAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Significant Group Concentrations Of Credit Risk [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MinimumCommercialRealEstateLoansAmountRequiredForAdditionalReview_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Minimum commercial real estate loans amount required for additional review</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MinimumCommercialRealEstateLoansAmountRequiredForAdditionalReview_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Minimum Commercial Real Estate Loans Amount Required For Additional Review</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_AdvertisingContractsMaturityDescription" xlink:type="locator" xlink:label="pbhc_AdvertisingContractsMaturityDescription" />
    <link:label xml:lang="en-US" xlink:label="pbhc_AdvertisingContractsMaturityDescription_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Advertising contracts maturity description.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_AdvertisingContractsMaturityDescription" xlink:to="pbhc_AdvertisingContractsMaturityDescription_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_MarketingAndAdvertisingExpenseAbstract" xlink:type="locator" xlink:label="us-gaap_MarketingAndAdvertisingExpenseAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_MarketingAndAdvertisingExpenseAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Advertising Expense [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_MarketingAndAdvertisingExpenseAbstract" xlink:to="us-gaap_MarketingAndAdvertisingExpenseAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_MarketingAndAdvertisingExpenseAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Marketing And Advertising Expense [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AdvertisingContractsMaturityDescription_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Advertising contracts maturity description</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AdvertisingContractsMaturityDescription_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Advertising Contracts Maturity Description</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_MaturityPeriodForClassificationAsCashAndCashEquivalentsMaximum" xlink:type="locator" xlink:label="pbhc_MaturityPeriodForClassificationAsCashAndCashEquivalentsMaximum" />
    <link:label xml:lang="en-US" xlink:label="pbhc_MaturityPeriodForClassificationAsCashAndCashEquivalentsMaximum_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The maximum maturity period for liquid resources to be classified as cash and cash equivalents.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_MaturityPeriodForClassificationAsCashAndCashEquivalentsMaximum" xlink:to="pbhc_MaturityPeriodForClassificationAsCashAndCashEquivalentsMaximum_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_CashAndCashEquivalentsAbstract" xlink:type="locator" xlink:label="us-gaap_CashAndCashEquivalentsAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CashAndCashEquivalentsAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Cash and Cash Equivalents [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_CashAndCashEquivalentsAbstract" xlink:to="us-gaap_CashAndCashEquivalentsAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CashAndCashEquivalentsAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Cash And Cash Equivalents [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MaturityPeriodForClassificationAsCashAndCashEquivalentsMaximum_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Maturity period for classification as cash and cash equivalents, maximum</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MaturityPeriodForClassificationAsCashAndCashEquivalentsMaximum_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Maturity Period For Classification As Cash And Cash Equivalents Maximum</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_MaximumPercentageForEstimatingSpecificAndGeneralLosses" xlink:type="locator" xlink:label="pbhc_MaximumPercentageForEstimatingSpecificAndGeneralLosses" />
    <link:label xml:lang="en-US" xlink:label="pbhc_MaximumPercentageForEstimatingSpecificAndGeneralLosses_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Maximum percentage for estimating specific and general losses in portfolio.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_MaximumPercentageForEstimatingSpecificAndGeneralLosses" xlink:to="pbhc_MaximumPercentageForEstimatingSpecificAndGeneralLosses_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_AllowanceForLoansLossesAbstract" xlink:type="locator" xlink:label="pbhc_AllowanceForLoansLossesAbstract" />
    <link:label xml:lang="en-US" xlink:label="pbhc_AllowanceForLoansLossesAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Allowance for Loan Losses [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_AllowanceForLoansLossesAbstract" xlink:to="pbhc_AllowanceForLoansLossesAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_AllowanceForLoansLossesAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Allowance for Loan Losses [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AllowanceForLoansLossesAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Allowance For Loans Losses [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MaximumPercentageForEstimatingSpecificAndGeneralLosses_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Maximum percentage for estimating specific and general losses</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MaximumPercentageForEstimatingSpecificAndGeneralLosses_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Maximum Percentage For Estimating Specific And General Losses</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_IndividualResidentialMortgageLoansThresholdForEvaluationsOfImpairment" xlink:type="locator" xlink:label="pbhc_IndividualResidentialMortgageLoansThresholdForEvaluationsOfImpairment" />
    <link:label xml:lang="en-US" xlink:label="pbhc_IndividualResidentialMortgageLoansThresholdForEvaluationsOfImpairment_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Minimum amount of individual residential mortgage loans, which the entity collectively evaluated for impairment.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_IndividualResidentialMortgageLoansThresholdForEvaluationsOfImpairment" xlink:to="pbhc_IndividualResidentialMortgageLoansThresholdForEvaluationsOfImpairment_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_IndividualResidentialMortgageLoansThresholdForEvaluationsOfImpairment_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Minimum amount, of residential mortgage loans threshold for evaluations of impairment</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_IndividualResidentialMortgageLoansThresholdForEvaluationsOfImpairment_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Individual Residential Mortgage Loans Threshold For Evaluations Of Impairment</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ConsecutiveMonthsOfCurrentPaymentsBeforeNonAccrualTroubledDebtRestructuredLoansAreRestoredToAccrualStatus" xlink:type="locator" xlink:label="pbhc_ConsecutiveMonthsOfCurrentPaymentsBeforeNonAccrualTroubledDebtRestructuredLoansAreRestoredToAccrualStatus" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ConsecutiveMonthsOfCurrentPaymentsBeforeNonAccrualTroubledDebtRestructuredLoansAreRestoredToAccrualStatus_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Consecutive period of current payments before non-accrual troubled debt restructured loans are restored to accrual status.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ConsecutiveMonthsOfCurrentPaymentsBeforeNonAccrualTroubledDebtRestructuredLoansAreRestoredToAccrualStatus" xlink:to="pbhc_ConsecutiveMonthsOfCurrentPaymentsBeforeNonAccrualTroubledDebtRestructuredLoansAreRestoredToAccrualStatus_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_IncomeRecognitionOnImpairedAndNonAccrualLoansAbstract" xlink:type="locator" xlink:label="pbhc_IncomeRecognitionOnImpairedAndNonAccrualLoansAbstract" />
    <link:label xml:lang="en-US" xlink:label="pbhc_IncomeRecognitionOnImpairedAndNonAccrualLoansAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Income Recognition on Impaired and Non-accrual Loans [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_IncomeRecognitionOnImpairedAndNonAccrualLoansAbstract" xlink:to="pbhc_IncomeRecognitionOnImpairedAndNonAccrualLoansAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_IncomeRecognitionOnImpairedAndNonAccrualLoansAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Income Recognition on Impaired and Non-accrual Loans [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_IncomeRecognitionOnImpairedAndNonAccrualLoansAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Income Recognition On Impaired And Non Accrual Loans [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ConsecutiveMonthsOfCurrentPaymentsBeforeNonAccrualTroubledDebtRestructuredLoansAreRestoredToAccrualStatus_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Number of consecutive months of current payments before non-accrual troubled debt restructured loans are restored to accrual status</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ConsecutiveMonthsOfCurrentPaymentsBeforeNonAccrualTroubledDebtRestructuredLoansAreRestoredToAccrualStatus_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Consecutive Months Of Current Payments Before Non Accrual Troubled Debt Restructured Loans Are Restored To Accrual Status</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_PropertyPlantAndEquipmentAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Premises and Equipment [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_PropertyPlantAndEquipmentUsefulLife" xlink:type="locator" xlink:label="us-gaap_PropertyPlantAndEquipmentUsefulLife" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PropertyPlantAndEquipmentUsefulLife_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Useful life</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_PropertyPlantAndEquipmentUsefulLife" xlink:to="us-gaap_PropertyPlantAndEquipmentUsefulLife_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PropertyPlantAndEquipmentUsefulLife_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Property Plant And Equipment Useful Life</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_GoodwillAndIntangibleAssetsDisclosureAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Goodwill and Intangible Assets [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FiniteLivedIntangibleAssetUsefulLife" xlink:type="locator" xlink:label="us-gaap_FiniteLivedIntangibleAssetUsefulLife" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FiniteLivedIntangibleAssetUsefulLife_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Estimated useful life</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FiniteLivedIntangibleAssetUsefulLife" xlink:to="us-gaap_FiniteLivedIntangibleAssetUsefulLife_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FiniteLivedIntangibleAssetUsefulLife_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Finite Lived Intangible Asset Useful Life</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DerivativeNumberOfInstrumentsHeld" xlink:type="locator" xlink:label="us-gaap_DerivativeNumberOfInstrumentsHeld" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DerivativeNumberOfInstrumentsHeld_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Number of interest rate derivatives</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DerivativeNumberOfInstrumentsHeld" xlink:to="us-gaap_DerivativeNumberOfInstrumentsHeld_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DerivativeNumberOfInstrumentsHeld_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Derivative Number Of Instruments Held</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_UnrealizedGainsAvailableForSaleSecurities" xlink:type="locator" xlink:label="pbhc_UnrealizedGainsAvailableForSaleSecurities" />
    <link:label xml:lang="en-US" xlink:label="pbhc_UnrealizedGainsAvailableForSaleSecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Accumulated appreciation or loss, before tax, in value of the total of available-for-sale securities at the end of an accounting period.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_UnrealizedGainsAvailableForSaleSecurities" xlink:to="pbhc_UnrealizedGainsAvailableForSaleSecurities_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_UnrealizedGainsOnAvailableForSaleSecuritiesTaxEffect" xlink:type="locator" xlink:label="pbhc_UnrealizedGainsOnAvailableForSaleSecuritiesTaxEffect" />
    <link:label xml:lang="en-US" xlink:label="pbhc_UnrealizedGainsOnAvailableForSaleSecuritiesTaxEffect_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Tax effect of unrealized gains on available-for-sale securities included in accumulated other comprehensive income (loss).</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_UnrealizedGainsOnAvailableForSaleSecuritiesTaxEffect" xlink:to="pbhc_UnrealizedGainsOnAvailableForSaleSecuritiesTaxEffect_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_UnrealizedHoldingLossesOnHedgingActivitiesDuringPeriod" xlink:type="locator" xlink:label="pbhc_UnrealizedHoldingLossesOnHedgingActivitiesDuringPeriod" />
    <link:label xml:lang="en-US" xlink:label="pbhc_UnrealizedHoldingLossesOnHedgingActivitiesDuringPeriod_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Unrealized holding losses on hedging activities during period</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_UnrealizedHoldingLossesOnHedgingActivitiesDuringPeriod" xlink:to="pbhc_UnrealizedHoldingLossesOnHedgingActivitiesDuringPeriod_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_UnrealizedHoldingLossesOnHedgingActivitiesTaxEffect" xlink:type="locator" xlink:label="pbhc_UnrealizedHoldingLossesOnHedgingActivitiesTaxEffect" />
    <link:label xml:lang="en-US" xlink:label="pbhc_UnrealizedHoldingLossesOnHedgingActivitiesTaxEffect_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Unrealized holding losses on hedging activities tax effect</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_UnrealizedHoldingLossesOnHedgingActivitiesTaxEffect" xlink:to="pbhc_UnrealizedHoldingLossesOnHedgingActivitiesTaxEffect_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_NetUnrealizedLossOnHedgingActivities" xlink:type="locator" xlink:label="pbhc_NetUnrealizedLossOnHedgingActivities" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NetUnrealizedLossOnHedgingActivities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Net unrealized loss on hedging activities</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_NetUnrealizedLossOnHedgingActivities" xlink:to="pbhc_NetUnrealizedLossOnHedgingActivities_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_UnrealizedLossOnSecuritiesTransferredToHeldToMaturity" xlink:type="locator" xlink:label="pbhc_UnrealizedLossOnSecuritiesTransferredToHeldToMaturity" />
    <link:label xml:lang="en-US" xlink:label="pbhc_UnrealizedLossOnSecuritiesTransferredToHeldToMaturity_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Accumulated appreciation or loss, before tax, in value of the total of securities transferred to held-to-maturity at the end of an accounting period.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_UnrealizedLossOnSecuritiesTransferredToHeldToMaturity" xlink:to="pbhc_UnrealizedLossOnSecuritiesTransferredToHeldToMaturity_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_UnrealizedLossOnSecuritiesTransferredToHeldToMaturityTaxEffect" xlink:type="locator" xlink:label="pbhc_UnrealizedLossOnSecuritiesTransferredToHeldToMaturityTaxEffect" />
    <link:label xml:lang="en-US" xlink:label="pbhc_UnrealizedLossOnSecuritiesTransferredToHeldToMaturityTaxEffect_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Tax effect of unrealized gains on securities transferred to held to maturity included in accumulated other comprehensive income (loss).</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_UnrealizedLossOnSecuritiesTransferredToHeldToMaturityTaxEffect" xlink:to="pbhc_UnrealizedLossOnSecuritiesTransferredToHeldToMaturityTaxEffect_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_OtherComprehensiveIncomeReclassificationAdjustmentForHeldToMaturityTransferredToAvailableForSaleSecuritiesNetOfTaxes" xlink:type="locator" xlink:label="pbhc_OtherComprehensiveIncomeReclassificationAdjustmentForHeldToMaturityTransferredToAvailableForSaleSecuritiesNetOfTaxes" />
    <link:label xml:lang="en-US" xlink:label="pbhc_OtherComprehensiveIncomeReclassificationAdjustmentForHeldToMaturityTransferredToAvailableForSaleSecuritiesNetOfTaxes_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Unrealized holding gain (loss), net of tax, at the date of the transfer for a debt security from the held-to-maturity category transferred into the available-for-sale category. Also includes the unrealized gain (loss) at the date of transfer for a debt security from the available-for-sale category transferred into the held-to-maturity category.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_OtherComprehensiveIncomeReclassificationAdjustmentForHeldToMaturityTransferredToAvailableForSaleSecuritiesNetOfTaxes" xlink:to="pbhc_OtherComprehensiveIncomeReclassificationAdjustmentForHeldToMaturityTransferredToAvailableForSaleSecuritiesNetOfTaxes_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AccumulatedOtherComprehensiveIncomeLossNetOfTaxAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">ACCUMULATED OTHER COMPREHENSIVE INCOME (LOSS) [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_UnrealizedLossAndTransitionObligationForPensionAndOtherPostretirementObligations_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Unrealized loss for pension and other postretirement obligations</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_UnrealizedLossAndTransitionObligationForPensionAndOtherPostretirementObligations_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Unrealized Loss And Transition Obligation For Pension And Other Postretirement Obligations</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_UnrealizedLossAndTransitionObligationForPensionAndOtherPostretirementObligationsTaxEffect_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Tax effect</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_UnrealizedLossAndTransitionObligationForPensionAndOtherPostretirementObligationsTaxEffect_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Unrealized Loss And Transition Obligation For Pension And Other Postretirement Obligations Tax Effect</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AccumulatedOtherComprehensiveIncomeLossDefinedBenefitPensionAndOtherPostretirementPlansNetOfTax" xlink:type="locator" xlink:label="us-gaap_AccumulatedOtherComprehensiveIncomeLossDefinedBenefitPensionAndOtherPostretirementPlansNetOfTax" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AccumulatedOtherComprehensiveIncomeLossDefinedBenefitPensionAndOtherPostretirementPlansNetOfTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedTotalLabel">Net unrealized loss for pension and other postretirement obligations</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AccumulatedOtherComprehensiveIncomeLossDefinedBenefitPensionAndOtherPostretirementPlansNetOfTax" xlink:to="us-gaap_AccumulatedOtherComprehensiveIncomeLossDefinedBenefitPensionAndOtherPostretirementPlansNetOfTax_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AccumulatedOtherComprehensiveIncomeLossDefinedBenefitPensionAndOtherPostretirementPlansNetOfTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Accumulated Other Comprehensive Income Loss Defined Benefit Pension And Other Postretirement Plans Net Of Tax</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_UnrealizedGainsAvailableForSaleSecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Unrealized gain (loss) on available-for-sale securities</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_UnrealizedGainsAvailableForSaleSecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Unrealized Gains Available For Sale Securities</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_UnrealizedGainsOnAvailableForSaleSecuritiesTaxEffect_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Tax effect</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_UnrealizedGainsOnAvailableForSaleSecuritiesTaxEffect_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Unrealized Gains On Available For Sale Securities Tax Effect</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AccumulatedOtherComprehensiveIncomeLossAvailableForSaleSecuritiesAdjustmentNetOfTax" xlink:type="locator" xlink:label="us-gaap_AccumulatedOtherComprehensiveIncomeLossAvailableForSaleSecuritiesAdjustmentNetOfTax" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AccumulatedOtherComprehensiveIncomeLossAvailableForSaleSecuritiesAdjustmentNetOfTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Net unrealized gain (loss) on available-for-sale securities</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AccumulatedOtherComprehensiveIncomeLossAvailableForSaleSecuritiesAdjustmentNetOfTax" xlink:to="us-gaap_AccumulatedOtherComprehensiveIncomeLossAvailableForSaleSecuritiesAdjustmentNetOfTax_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AccumulatedOtherComprehensiveIncomeLossAvailableForSaleSecuritiesAdjustmentNetOfTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Accumulated Other Comprehensive Income Loss Available For Sale Securities Adjustment Net Of Tax</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_UnrealizedHoldingLossesOnHedgingActivitiesDuringPeriod_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Unrealized holding losses on hedging activities arising during the period</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_UnrealizedHoldingLossesOnHedgingActivitiesDuringPeriod_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Unrealized Holding Losses On Hedging Activities During Period</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_UnrealizedHoldingLossesOnHedgingActivitiesTaxEffect_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Tax effect</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_UnrealizedHoldingLossesOnHedgingActivitiesTaxEffect_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Unrealized Holding Losses On Hedging Activities Tax Effect</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NetUnrealizedLossOnHedgingActivities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Net unrealized loss on hedging activities</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NetUnrealizedLossOnHedgingActivities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Net Unrealized Loss On Hedging Activities</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_UnrealizedLossOnSecuritiesTransferredToHeldToMaturity_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Unrealized loss on securities transferred to held-to-maturity</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_UnrealizedLossOnSecuritiesTransferredToHeldToMaturity_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Unrealized Loss On Securities Transferred To Held To Maturity</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_UnrealizedLossOnSecuritiesTransferredToHeldToMaturityTaxEffect_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Tax effect</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_UnrealizedLossOnSecuritiesTransferredToHeldToMaturityTaxEffect_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Unrealized Loss On Securities Transferred To Held To Maturity Tax Effect</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_OtherComprehensiveIncomeReclassificationAdjustmentForHeldToMaturityTransferredToAvailableForSaleSecuritiesNetOfTaxes_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Net unrealized loss on securities transferred to held-to-maturity</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_OtherComprehensiveIncomeReclassificationAdjustmentForHeldToMaturityTransferredToAvailableForSaleSecuritiesNetOfTaxes_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Other Comprehensive Income Reclassification Adjustment For Held To Maturity Transferred To Available For Sale Securities Net Of Taxes</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_AccumulatedOtherComprehensiveIncomeLossNetOfTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Accumulated other comprehensive loss</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ScheduleOfAntidilutiveSecuritiesExcludedFromComputationOfEarningsPerShareTable" xlink:type="locator" xlink:label="us-gaap_ScheduleOfAntidilutiveSecuritiesExcludedFromComputationOfEarningsPerShareTable" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfAntidilutiveSecuritiesExcludedFromComputationOfEarningsPerShareTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Schedule Of Antidilutive Securities Excluded From Computation Of Earnings Per Share [Table]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ScheduleOfAntidilutiveSecuritiesExcludedFromComputationOfEarningsPerShareTable" xlink:to="us-gaap_ScheduleOfAntidilutiveSecuritiesExcludedFromComputationOfEarningsPerShareTable_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfAntidilutiveSecuritiesExcludedFromComputationOfEarningsPerShareTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Antidilutive Securities Excluded From Computation Of Earnings Per Share [Table]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AntidilutiveSecuritiesExcludedFromComputationOfEarningsPerShareByAntidilutiveSecuritiesAxis" xlink:type="locator" xlink:label="us-gaap_AntidilutiveSecuritiesExcludedFromComputationOfEarningsPerShareByAntidilutiveSecuritiesAxis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AntidilutiveSecuritiesExcludedFromComputationOfEarningsPerShareByAntidilutiveSecuritiesAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Antidilutive Securities</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AntidilutiveSecuritiesExcludedFromComputationOfEarningsPerShareByAntidilutiveSecuritiesAxis" xlink:to="us-gaap_AntidilutiveSecuritiesExcludedFromComputationOfEarningsPerShareByAntidilutiveSecuritiesAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AntidilutiveSecuritiesExcludedFromComputationOfEarningsPerShareByAntidilutiveSecuritiesAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Antidilutive Securities Excluded From Computation Of Earnings Per Share By Antidilutive Securities [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AntidilutiveSecuritiesNameDomain" xlink:type="locator" xlink:label="us-gaap_AntidilutiveSecuritiesNameDomain" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AntidilutiveSecuritiesNameDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Antidilutive Securities, Name</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AntidilutiveSecuritiesNameDomain" xlink:to="us-gaap_AntidilutiveSecuritiesNameDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AntidilutiveSecuritiesNameDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Antidilutive Securities Name [Domain]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_EmployeeStockOptionMember" xlink:type="locator" xlink:label="us-gaap_EmployeeStockOptionMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EmployeeStockOptionMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Stock Option [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_EmployeeStockOptionMember" xlink:to="us-gaap_EmployeeStockOptionMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EmployeeStockOptionMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Employee Stock Option [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AntidilutiveSecuritiesExcludedFromComputationOfEarningsPerShareLineItems" xlink:type="locator" xlink:label="us-gaap_AntidilutiveSecuritiesExcludedFromComputationOfEarningsPerShareLineItems" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AntidilutiveSecuritiesExcludedFromComputationOfEarningsPerShareLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Antidilutive Securities Excluded From Computation Of Earnings Per Share [Line Items]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AntidilutiveSecuritiesExcludedFromComputationOfEarningsPerShareLineItems" xlink:to="us-gaap_AntidilutiveSecuritiesExcludedFromComputationOfEarningsPerShareLineItems_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AntidilutiveSecuritiesExcludedFromComputationOfEarningsPerShareLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Antidilutive Securities Excluded From Computation Of Earnings Per Share [Line Items]</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_CommonStockParOrStatedValuePerShare_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Common stock, par value (in dollars per share)</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_SharesIssuedPricePerShare" xlink:type="locator" xlink:label="us-gaap_SharesIssuedPricePerShare" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SharesIssuedPricePerShare_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Purchase price</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_SharesIssuedPricePerShare" xlink:to="us-gaap_SharesIssuedPricePerShare_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SharesIssuedPricePerShare_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Shares Issued Price Per Share</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_PreferredStockParOrStatedValuePerShare_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Preferred stock, par value (in dollars per share)</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_WarrantsAndRightsOutstanding" xlink:type="locator" xlink:label="us-gaap_WarrantsAndRightsOutstanding" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_WarrantsAndRightsOutstanding_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Fair value of warrants</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_WarrantsAndRightsOutstanding" xlink:to="us-gaap_WarrantsAndRightsOutstanding_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_WarrantsAndRightsOutstanding_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Warrants And Rights Outstanding</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ClassOfWarrantOrRightExercisePriceOfWarrantsOrRights1" xlink:type="locator" xlink:label="us-gaap_ClassOfWarrantOrRightExercisePriceOfWarrantsOrRights1" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ClassOfWarrantOrRightExercisePriceOfWarrantsOrRights1_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Exercise price of warrant</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ClassOfWarrantOrRightExercisePriceOfWarrantsOrRights1" xlink:to="us-gaap_ClassOfWarrantOrRightExercisePriceOfWarrantsOrRights1_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ClassOfWarrantOrRightExercisePriceOfWarrantsOrRights1_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Class Of Warrant Or Right Exercise Price Of Warrants Or Rights1</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AntidilutiveSecuritiesExcludedFromComputationOfEarningsPerShareAmount" xlink:type="locator" xlink:label="us-gaap_AntidilutiveSecuritiesExcludedFromComputationOfEarningsPerShareAmount" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AntidilutiveSecuritiesExcludedFromComputationOfEarningsPerShareAmount_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Anti-dilutive stock options (in shares)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AntidilutiveSecuritiesExcludedFromComputationOfEarningsPerShareAmount" xlink:to="us-gaap_AntidilutiveSecuritiesExcludedFromComputationOfEarningsPerShareAmount_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AntidilutiveSecuritiesExcludedFromComputationOfEarningsPerShareAmount_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Antidilutive Securities Excluded From Computation Of Earnings Per Share Amount</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_WeightedAverageNumberOfSharesOutstandingBasic" xlink:type="locator" xlink:label="us-gaap_WeightedAverageNumberOfSharesOutstandingBasic" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_WeightedAverageNumberOfSharesOutstandingBasic_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Basic weighted average common shares outstanding (in shares)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_WeightedAverageNumberOfSharesOutstandingBasic" xlink:to="us-gaap_WeightedAverageNumberOfSharesOutstandingBasic_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_WeightedAverageNumberOfSharesOutstandingBasic_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Weighted Average Number Of Shares Outstanding Basic</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_IncrementalCommonSharesAttributableToShareBasedPaymentArrangements" xlink:type="locator" xlink:label="us-gaap_IncrementalCommonSharesAttributableToShareBasedPaymentArrangements" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncrementalCommonSharesAttributableToShareBasedPaymentArrangements_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Effect of assumed exercise of stock options and unvested restricted stock units (in shares)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_IncrementalCommonSharesAttributableToShareBasedPaymentArrangements" xlink:to="us-gaap_IncrementalCommonSharesAttributableToShareBasedPaymentArrangements_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncrementalCommonSharesAttributableToShareBasedPaymentArrangements_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Incremental Common Shares Attributable To Share Based Payment Arrangements</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_WeightedAverageNumberOfDilutedSharesOutstanding" xlink:type="locator" xlink:label="us-gaap_WeightedAverageNumberOfDilutedSharesOutstanding" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_WeightedAverageNumberOfDilutedSharesOutstanding_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Diluted weighted average common shares outstanding (in shares)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_WeightedAverageNumberOfDilutedSharesOutstanding" xlink:to="us-gaap_WeightedAverageNumberOfDilutedSharesOutstanding_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_WeightedAverageNumberOfDilutedSharesOutstanding_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Weighted Average Number Of Diluted Shares Outstanding</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_EarningsPerShareBasic_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Earnings per common share - basic</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_AvailableForSaleDebtAndEquitySecuritiesAmortizedCost" xlink:type="locator" xlink:label="pbhc_AvailableForSaleDebtAndEquitySecuritiesAmortizedCost" />
    <link:label xml:lang="en-US" xlink:label="pbhc_AvailableForSaleDebtAndEquitySecuritiesAmortizedCost_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Available for sale debt and equity securities amortized cost.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_AvailableForSaleDebtAndEquitySecuritiesAmortizedCost" xlink:to="pbhc_AvailableForSaleDebtAndEquitySecuritiesAmortizedCost_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ScheduleOfAvailableForSaleSecuritiesAndHeldToMaturitySecuritiesLineItems" xlink:type="locator" xlink:label="pbhc_ScheduleOfAvailableForSaleSecuritiesAndHeldToMaturitySecuritiesLineItems" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ScheduleOfAvailableForSaleSecuritiesAndHeldToMaturitySecuritiesLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Schedule of available for sale securities and held to maturity securities.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ScheduleOfAvailableForSaleSecuritiesAndHeldToMaturitySecuritiesLineItems" xlink:to="pbhc_ScheduleOfAvailableForSaleSecuritiesAndHeldToMaturitySecuritiesLineItems_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ScheduleOfAvailableForSaleSecuritiesAndHeldToMaturitySecuritiesTable" xlink:type="locator" xlink:label="pbhc_ScheduleOfAvailableForSaleSecuritiesAndHeldToMaturitySecuritiesTable" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ScheduleOfAvailableForSaleSecuritiesAndHeldToMaturitySecuritiesTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Schedule of available for sale securities and held to maturity securities.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ScheduleOfAvailableForSaleSecuritiesAndHeldToMaturitySecuritiesTable" xlink:to="pbhc_ScheduleOfAvailableForSaleSecuritiesAndHeldToMaturitySecuritiesTable_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ScheduleOfAvailableForSaleSecuritiesAndHeldToMaturitySecuritiesTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Schedule Of Available For Sale Securities And Held To Maturity Securities [Table]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ScheduleOfAvailableForSaleSecuritiesAndHeldToMaturitySecuritiesTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Available For Sale Securities And Held To Maturity Securities [Table]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FinancialInstrumentAxis" xlink:type="locator" xlink:label="us-gaap_FinancialInstrumentAxis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancialInstrumentAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Financial Instrument</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FinancialInstrumentAxis" xlink:to="us-gaap_FinancialInstrumentAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancialInstrumentAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Financial Instrument [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_TransfersAndServicingOfFinancialInstrumentsTypesOfFinancialInstrumentsDomain" xlink:type="locator" xlink:label="us-gaap_TransfersAndServicingOfFinancialInstrumentsTypesOfFinancialInstrumentsDomain" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TransfersAndServicingOfFinancialInstrumentsTypesOfFinancialInstrumentsDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Financial Instruments</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_TransfersAndServicingOfFinancialInstrumentsTypesOfFinancialInstrumentsDomain" xlink:to="us-gaap_TransfersAndServicingOfFinancialInstrumentsTypesOfFinancialInstrumentsDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TransfersAndServicingOfFinancialInstrumentsTypesOfFinancialInstrumentsDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Transfers And Servicing Of Financial Instruments Types Of Financial Instruments [Domain]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_USTreasuryAndGovernmentMember" xlink:type="locator" xlink:label="us-gaap_USTreasuryAndGovernmentMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_USTreasuryAndGovernmentMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">US Treasury, Agencies and GSEs [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_USTreasuryAndGovernmentMember" xlink:to="us-gaap_USTreasuryAndGovernmentMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_USTreasuryAndGovernmentMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">U S Treasury And Government [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_USStatesAndPoliticalSubdivisionsMember" xlink:type="locator" xlink:label="us-gaap_USStatesAndPoliticalSubdivisionsMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_USStatesAndPoliticalSubdivisionsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">State and Political Subdivisions [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_USStatesAndPoliticalSubdivisionsMember" xlink:to="us-gaap_USStatesAndPoliticalSubdivisionsMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_USStatesAndPoliticalSubdivisionsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">U S States And Political Subdivisions [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_CorporateDebtSecuritiesMember" xlink:type="locator" xlink:label="us-gaap_CorporateDebtSecuritiesMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CorporateDebtSecuritiesMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Corporate [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_CorporateDebtSecuritiesMember" xlink:to="us-gaap_CorporateDebtSecuritiesMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CorporateDebtSecuritiesMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Corporate Debt Securities [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AssetBackedSecuritiesMember" xlink:type="locator" xlink:label="us-gaap_AssetBackedSecuritiesMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AssetBackedSecuritiesMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Asset Backed Securities [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AssetBackedSecuritiesMember" xlink:to="us-gaap_AssetBackedSecuritiesMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AssetBackedSecuritiesMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Asset Backed Securities [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ResidentialMortgageBackedSecuritiesMember" xlink:type="locator" xlink:label="us-gaap_ResidentialMortgageBackedSecuritiesMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ResidentialMortgageBackedSecuritiesMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Residential Mortgage-Backed - US Agency [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ResidentialMortgageBackedSecuritiesMember" xlink:to="us-gaap_ResidentialMortgageBackedSecuritiesMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ResidentialMortgageBackedSecuritiesMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Residential Mortgage Backed Securities [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_MortgageBackedSecuritiesIssuedByUSGovernmentSponsoredEnterprisesMember" xlink:type="locator" xlink:label="us-gaap_MortgageBackedSecuritiesIssuedByUSGovernmentSponsoredEnterprisesMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_MortgageBackedSecuritiesIssuedByUSGovernmentSponsoredEnterprisesMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Collateralized Mortgage Obligations - US Agency [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_MortgageBackedSecuritiesIssuedByUSGovernmentSponsoredEnterprisesMember" xlink:to="us-gaap_MortgageBackedSecuritiesIssuedByUSGovernmentSponsoredEnterprisesMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_MortgageBackedSecuritiesIssuedByUSGovernmentSponsoredEnterprisesMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Mortgage Backed Securities Issued By U S Government Sponsored Enterprises [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_MortgageBackedSecuritiesIssuedByPrivateEnterprisesMember" xlink:type="locator" xlink:label="us-gaap_MortgageBackedSecuritiesIssuedByPrivateEnterprisesMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_MortgageBackedSecuritiesIssuedByPrivateEnterprisesMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Collateralized Mortgage Obligations - Private Label [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_MortgageBackedSecuritiesIssuedByPrivateEnterprisesMember" xlink:to="us-gaap_MortgageBackedSecuritiesIssuedByPrivateEnterprisesMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_MortgageBackedSecuritiesIssuedByPrivateEnterprisesMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Mortgage Backed Securities Issued By Private Enterprises [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_MutualFundsUltraShortMortgageFundsMember" xlink:type="locator" xlink:label="pbhc_MutualFundsUltraShortMortgageFundsMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_MutualFundsUltraShortMortgageFundsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">This category includes information about ultra short mortgage funds.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_MutualFundsUltraShortMortgageFundsMember" xlink:to="pbhc_MutualFundsUltraShortMortgageFundsMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_MutualFundsUltraShortMortgageFundsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Mutual Funds Ultra Short Mortgage Fund [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MutualFundsUltraShortMortgageFundsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Mutual Funds Ultra Short Mortgage Funds [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_MutualFundsCommonStockFinancialServicesIndustryMember" xlink:type="locator" xlink:label="pbhc_MutualFundsCommonStockFinancialServicesIndustryMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_MutualFundsCommonStockFinancialServicesIndustryMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">This category includes information about common stock financial services industry.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_MutualFundsCommonStockFinancialServicesIndustryMember" xlink:to="pbhc_MutualFundsCommonStockFinancialServicesIndustryMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_MutualFundsCommonStockFinancialServicesIndustryMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Mutual funds Common Stock Financial Services Industry [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MutualFundsCommonStockFinancialServicesIndustryMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Mutual Funds Common Stock Financial Services Industry [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ScheduleOfAvailableForSaleSecuritiesAndHeldToMaturitySecuritiesLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Schedule of Available-for-sale Securities and Held-to-Maturity Securities [Line Items]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ScheduleOfAvailableForSaleSecuritiesAndHeldToMaturitySecuritiesLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Available For Sale Securities And Held To Maturity Securities [Line Items]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AvailableForSaleDebtSecuritiesAmortizedCostBasis" xlink:type="locator" xlink:label="us-gaap_AvailableForSaleDebtSecuritiesAmortizedCostBasis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AvailableForSaleDebtSecuritiesAmortizedCostBasis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Totals, amortized cost</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AvailableForSaleDebtSecuritiesAmortizedCostBasis" xlink:to="us-gaap_AvailableForSaleDebtSecuritiesAmortizedCostBasis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AvailableForSaleDebtSecuritiesAmortizedCostBasis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Available For Sale Debt Securities Amortized Cost Basis</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AvailableForSaleDebtSecuritiesAccumulatedGrossUnrealizedGainBeforeTax" xlink:type="locator" xlink:label="us-gaap_AvailableForSaleDebtSecuritiesAccumulatedGrossUnrealizedGainBeforeTax" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AvailableForSaleDebtSecuritiesAccumulatedGrossUnrealizedGainBeforeTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Gross Unrealized Gains, Debt investment securities</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AvailableForSaleDebtSecuritiesAccumulatedGrossUnrealizedGainBeforeTax" xlink:to="us-gaap_AvailableForSaleDebtSecuritiesAccumulatedGrossUnrealizedGainBeforeTax_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AvailableForSaleDebtSecuritiesAccumulatedGrossUnrealizedGainBeforeTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Available For Sale Debt Securities Accumulated Gross Unrealized Gain Before Tax</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AvailableForSaleDebtSecuritiesAccumulatedGrossUnrealizedLossBeforeTax" xlink:type="locator" xlink:label="us-gaap_AvailableForSaleDebtSecuritiesAccumulatedGrossUnrealizedLossBeforeTax" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AvailableForSaleDebtSecuritiesAccumulatedGrossUnrealizedLossBeforeTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Gross Unrealized Losses, Debt investment securities</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AvailableForSaleDebtSecuritiesAccumulatedGrossUnrealizedLossBeforeTax" xlink:to="us-gaap_AvailableForSaleDebtSecuritiesAccumulatedGrossUnrealizedLossBeforeTax_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AvailableForSaleDebtSecuritiesAccumulatedGrossUnrealizedLossBeforeTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Available For Sale Debt Securities Accumulated Gross Unrealized Loss Before Tax</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AvailableForSaleSecuritiesDebtSecurities" xlink:type="locator" xlink:label="us-gaap_AvailableForSaleSecuritiesDebtSecurities" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AvailableForSaleSecuritiesDebtSecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Available-for-sale Securities, Debt investment securities</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AvailableForSaleSecuritiesDebtSecurities" xlink:to="us-gaap_AvailableForSaleSecuritiesDebtSecurities_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AvailableForSaleSecuritiesDebtSecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Available For Sale Securities Debt Securities</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_EquitySecuritiesFvNiCost" xlink:type="locator" xlink:label="us-gaap_EquitySecuritiesFvNiCost" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EquitySecuritiesFvNiCost_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Amortized cost, equity securities</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_EquitySecuritiesFvNiCost" xlink:to="us-gaap_EquitySecuritiesFvNiCost_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EquitySecuritiesFvNiCost_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Equity Securities Fv Ni Cost</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_EquitySecuritiesAvailableForSaleAccumulatedGrossUnrealizedGainBeforeTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Gross Unrealized Gains, Equity investment securities</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_EquitySecuritiesAvailableForSaleAccumulatedGrossUnrealizedGainBeforeTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Equity Securities Available For Sale Accumulated Gross Unrealized Gain Before Tax</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_EquitySecuritiesAvailableForSaleAccumulatedGrossUnrealizedLossBeforeTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Gross Unrealized Losses, Equity investment securities</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_EquitySecuritiesAvailableForSaleAccumulatedGrossUnrealizedLossBeforeTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Equity Securities Available For Sale Accumulated Gross Unrealized Loss Before Tax</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_EquitySecuritiesFvNi" xlink:type="locator" xlink:label="us-gaap_EquitySecuritiesFvNi" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EquitySecuritiesFvNi_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Available-for-sale Securities, Equity investment securities</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_EquitySecuritiesFvNi" xlink:to="us-gaap_EquitySecuritiesFvNi_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EquitySecuritiesFvNi_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Equity Securities Fv Ni</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AvailableForSaleDebtAndEquitySecuritiesAmortizedCost_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Total investment securities, amortized cost basis</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AvailableForSaleDebtAndEquitySecuritiesAmortizedCost_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Available For Sale Debt And Equity Securities Amortized Cost</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AvailableForSaleDebtAndEquitySecuritiesAccumulatedGrossUnrealizedGainBeforeTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Gross Unrealized Gains</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AvailableForSaleDebtAndEquitySecuritiesAccumulatedGrossUnrealizedGainBeforeTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Available For Sale Debt And Equity Securities Accumulated Gross Unrealized Gain Before Tax</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AvailableForSaleDebtAndEquitySecuritiesAccumulatedGrossUnrealizedLossBeforeTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Gross Unrealized Losses</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AvailableForSaleDebtAndEquitySecuritiesAccumulatedGrossUnrealizedLossBeforeTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Available For Sale Debt And Equity Securities Accumulated Gross Unrealized Loss Before Tax</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AvailableForSaleDebtAndEquitySecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Estimated Fair Value</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Held-to-maturity securities, debt maturities, Amortized Cost</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_HeldToMaturitySecuritiesAccumulatedUnrecognizedHoldingGain" xlink:type="locator" xlink:label="us-gaap_HeldToMaturitySecuritiesAccumulatedUnrecognizedHoldingGain" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesAccumulatedUnrecognizedHoldingGain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Held to maturity, gross unrealized gains</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_HeldToMaturitySecuritiesAccumulatedUnrecognizedHoldingGain" xlink:to="us-gaap_HeldToMaturitySecuritiesAccumulatedUnrecognizedHoldingGain_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesAccumulatedUnrecognizedHoldingGain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Held To Maturity Securities Accumulated Unrecognized Holding Gain</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_HeldToMaturitySecuritiesAccumulatedUnrecognizedHoldingLoss" xlink:type="locator" xlink:label="us-gaap_HeldToMaturitySecuritiesAccumulatedUnrecognizedHoldingLoss" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesAccumulatedUnrecognizedHoldingLoss_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Held to maturity, gross unrealized losses</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_HeldToMaturitySecuritiesAccumulatedUnrecognizedHoldingLoss" xlink:to="us-gaap_HeldToMaturitySecuritiesAccumulatedUnrecognizedHoldingLoss_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesAccumulatedUnrecognizedHoldingLoss_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Held To Maturity Securities Accumulated Unrecognized Holding Loss</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesFairValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Held-to-maturity Securities, Debt Maturities, Fair Value</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_NumberOfSecurities" xlink:type="locator" xlink:label="pbhc_NumberOfSecurities" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfSecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">This element represents number of securities issued in investment securities.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_NumberOfSecurities" xlink:to="pbhc_NumberOfSecurities_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_InvestmentSecuritiesBookValue" xlink:type="locator" xlink:label="pbhc_InvestmentSecuritiesBookValue" />
    <link:label xml:lang="en-US" xlink:label="pbhc_InvestmentSecuritiesBookValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Investment under secured debt book value net.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_InvestmentSecuritiesBookValue" xlink:to="pbhc_InvestmentSecuritiesBookValue_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_CollateralizedMortgageObligationsOfPrivateLabelMember" xlink:type="locator" xlink:label="pbhc_CollateralizedMortgageObligationsOfPrivateLabelMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_CollateralizedMortgageObligationsOfPrivateLabelMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The category includes multiclass, pay-through securitizations collateralized by mortgages secured by assets, such as automobiles and boats, which are generally structured so that the collections of principal and interest due from the underlying debtors are paid to the holders of the CMO.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_CollateralizedMortgageObligationsOfPrivateLabelMember" xlink:to="pbhc_CollateralizedMortgageObligationsOfPrivateLabelMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_CollateralizedMortgageObligationsOfPrivateLabelMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Collateralized Mortgage Obligations of Private Label [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_CollateralizedMortgageObligationsOfPrivateLabelMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Collateralized Mortgage Obligations Of Private Label [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_FiftyFourPrivateLabelMortgageBackedSecuritiesMember" xlink:type="locator" xlink:label="pbhc_FiftyFourPrivateLabelMortgageBackedSecuritiesMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_FiftyFourPrivateLabelMortgageBackedSecuritiesMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Fifty Four Private Label Mortgage Backed Securities.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_FiftyFourPrivateLabelMortgageBackedSecuritiesMember" xlink:to="pbhc_FiftyFourPrivateLabelMortgageBackedSecuritiesMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_FiftyFourPrivateLabelMortgageBackedSecuritiesMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">54 Private-Label Mortgage-Backed Securities [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_FiftyFourPrivateLabelMortgageBackedSecuritiesMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Fifty Four Private Label Mortgage Backed Securities [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_TwentyTwoPrivateLabelAssetBackedSecuritiesMember" xlink:type="locator" xlink:label="pbhc_TwentyTwoPrivateLabelAssetBackedSecuritiesMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_TwentyTwoPrivateLabelAssetBackedSecuritiesMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Twenty two private label asset backed securities.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_TwentyTwoPrivateLabelAssetBackedSecuritiesMember" xlink:to="pbhc_TwentyTwoPrivateLabelAssetBackedSecuritiesMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_TwentyTwoPrivateLabelAssetBackedSecuritiesMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">22 Private-Label Asset - Backed Securities [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_TwentyTwoPrivateLabelAssetBackedSecuritiesMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Twenty Two Private Label Asset Backed Securities [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ThirtyThreePrivateLabelMortgageBackedSecuritiesMember" xlink:type="locator" xlink:label="pbhc_ThirtyThreePrivateLabelMortgageBackedSecuritiesMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ThirtyThreePrivateLabelMortgageBackedSecuritiesMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Thirty three private label mortgage backed securities.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ThirtyThreePrivateLabelMortgageBackedSecuritiesMember" xlink:to="pbhc_ThirtyThreePrivateLabelMortgageBackedSecuritiesMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ThirtyThreePrivateLabelMortgageBackedSecuritiesMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">33 Private-Label Mortgage-Backed Securities [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ThirtyThreePrivateLabelMortgageBackedSecuritiesMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Thirty Three Private Label Mortgage Backed Securities [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_TwentyThreePrivateLabelAssetBackedSecuritiesMember" xlink:type="locator" xlink:label="pbhc_TwentyThreePrivateLabelAssetBackedSecuritiesMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_TwentyThreePrivateLabelAssetBackedSecuritiesMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Twenty three private label asset backed securities.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_TwentyThreePrivateLabelAssetBackedSecuritiesMember" xlink:to="pbhc_TwentyThreePrivateLabelAssetBackedSecuritiesMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_TwentyThreePrivateLabelAssetBackedSecuritiesMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">23 Private-Label Asset - Backed Securities [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_TwentyThreePrivateLabelAssetBackedSecuritiesMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Twenty Three Private Label Asset Backed Securities [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_MunicipalBondsMember" xlink:type="locator" xlink:label="us-gaap_MunicipalBondsMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_MunicipalBondsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Municipal Bonds</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_MunicipalBondsMember" xlink:to="us-gaap_MunicipalBondsMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_MunicipalBondsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Municipal Bonds [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_PrivateLabelAssetsBackedSecuritiesRatedMember" xlink:type="locator" xlink:label="pbhc_PrivateLabelAssetsBackedSecuritiesRatedMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_PrivateLabelAssetsBackedSecuritiesRatedMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">private label assets backed securities rated.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_PrivateLabelAssetsBackedSecuritiesRatedMember" xlink:to="pbhc_PrivateLabelAssetsBackedSecuritiesRatedMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_PrivateLabelAssetsBackedSecuritiesRatedMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Fixed Rate, Private-label Asset-backed Security Rated [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_PrivateLabelAssetsBackedSecuritiesRatedMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Private Label Assets Backed Securities Rated [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_PrivateLabelMortgagedBackedSecuritiesNotRatedMember" xlink:type="locator" xlink:label="pbhc_PrivateLabelMortgagedBackedSecuritiesNotRatedMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_PrivateLabelMortgagedBackedSecuritiesNotRatedMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Private label mortgaged backed securities not rated.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_PrivateLabelMortgagedBackedSecuritiesNotRatedMember" xlink:to="pbhc_PrivateLabelMortgagedBackedSecuritiesNotRatedMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_PrivateLabelMortgagedBackedSecuritiesNotRatedMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Private Label Mortgaged Backed Securities Not Rated</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_PrivateLabelMortgagedBackedSecuritiesNotRatedMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Private Label Mortgaged Backed Securities Not Rated [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_OnePrivateLabelAssetBackedSecuritiesBySmallBusinessLoansMember" xlink:type="locator" xlink:label="pbhc_OnePrivateLabelAssetBackedSecuritiesBySmallBusinessLoansMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_OnePrivateLabelAssetBackedSecuritiesBySmallBusinessLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">One private label asset backed securities by small business loans</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_OnePrivateLabelAssetBackedSecuritiesBySmallBusinessLoansMember" xlink:to="pbhc_OnePrivateLabelAssetBackedSecuritiesBySmallBusinessLoansMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_OnePrivateLabelAssetBackedSecuritiesBySmallBusinessLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">1 Private-Label Asset - Backed Securities by Small Business Loan [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_OnePrivateLabelAssetBackedSecuritiesBySmallBusinessLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">One Private Label Asset Backed Securities By Small Business Loans [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_OnePrivateLabelAssetBackedSecuritiesByPrivateIssueStudentLoansMember" xlink:type="locator" xlink:label="pbhc_OnePrivateLabelAssetBackedSecuritiesByPrivateIssueStudentLoansMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_OnePrivateLabelAssetBackedSecuritiesByPrivateIssueStudentLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">One Private Label Asset Backed Securities By Private Issue Student Loans.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_OnePrivateLabelAssetBackedSecuritiesByPrivateIssueStudentLoansMember" xlink:to="pbhc_OnePrivateLabelAssetBackedSecuritiesByPrivateIssueStudentLoansMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_OnePrivateLabelAssetBackedSecuritiesByPrivateIssueStudentLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">1 Private-Label Asset - Backed Securities by Private-Issue Student Loan [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_OnePrivateLabelAssetBackedSecuritiesByPrivateIssueStudentLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">One Private Label Asset Backed Securities By Private Issue Student Loans [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_OnePrivateLabelAssetBackedSecuritiesByFederallyInsuredStudentLoansMember" xlink:type="locator" xlink:label="pbhc_OnePrivateLabelAssetBackedSecuritiesByFederallyInsuredStudentLoansMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_OnePrivateLabelAssetBackedSecuritiesByFederallyInsuredStudentLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">One private label asset backed securities by federally insured student loans.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_OnePrivateLabelAssetBackedSecuritiesByFederallyInsuredStudentLoansMember" xlink:to="pbhc_OnePrivateLabelAssetBackedSecuritiesByFederallyInsuredStudentLoansMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_OnePrivateLabelAssetBackedSecuritiesByFederallyInsuredStudentLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">1 Private-Label Asset - Backed Securities by Federally Insured Student Loan [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_OnePrivateLabelAssetBackedSecuritiesByFederallyInsuredStudentLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">One Private Label Asset Backed Securities By Federally Insured Student Loans [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_OnePrivateLabelCollateralizedMortgageObligationSecuritiesByResidentialMortgageLoansMember" xlink:type="locator" xlink:label="pbhc_OnePrivateLabelCollateralizedMortgageObligationSecuritiesByResidentialMortgageLoansMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_OnePrivateLabelCollateralizedMortgageObligationSecuritiesByResidentialMortgageLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">One private label collateralized mortgage obligation securities by residential mortgage loans.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_OnePrivateLabelCollateralizedMortgageObligationSecuritiesByResidentialMortgageLoansMember" xlink:to="pbhc_OnePrivateLabelCollateralizedMortgageObligationSecuritiesByResidentialMortgageLoansMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_OnePrivateLabelCollateralizedMortgageObligationSecuritiesByResidentialMortgageLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">One Private Label Collateralized Mortgage Obligation Securities By Residential Mortgage Loans</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_OnePrivateLabelCollateralizedMortgageObligationSecuritiesByResidentialMortgageLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">One Private Label Collateralized Mortgage Obligation Securities By Residential Mortgage Loans [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_InternalCreditAssessmentAxis" xlink:type="locator" xlink:label="us-gaap_InternalCreditAssessmentAxis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InternalCreditAssessmentAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Internal Credit Assessment</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_InternalCreditAssessmentAxis" xlink:to="us-gaap_InternalCreditAssessmentAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InternalCreditAssessmentAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Internal Credit Assessment [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_InternalCreditAssessmentDomain" xlink:type="locator" xlink:label="us-gaap_InternalCreditAssessmentDomain" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InternalCreditAssessmentDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Internal Credit Assessment</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_InternalCreditAssessmentDomain" xlink:to="us-gaap_InternalCreditAssessmentDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InternalCreditAssessmentDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Internal Credit Assessment [Domain]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_NRSROPMember" xlink:type="locator" xlink:label="pbhc_NRSROPMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NRSROPMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">NRSROP.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_NRSROPMember" xlink:to="pbhc_NRSROPMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NRSROPMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">N R S R O P</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NRSROPMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">N R S R O P [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AvailableForSaleSecuritiesContinuousUnrealizedLossPositionFairValueAbstract" xlink:type="locator" xlink:label="us-gaap_AvailableForSaleSecuritiesContinuousUnrealizedLossPositionFairValueAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AvailableForSaleSecuritiesContinuousUnrealizedLossPositionFairValueAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Available-for-sale Securities, Continuous Unrealized Loss Position, Fair Value [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AvailableForSaleSecuritiesContinuousUnrealizedLossPositionFairValueAbstract" xlink:to="us-gaap_AvailableForSaleSecuritiesContinuousUnrealizedLossPositionFairValueAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AvailableForSaleSecuritiesContinuousUnrealizedLossPositionFairValueAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Available For Sale Securities Continuous Unrealized Loss Position Fair Value [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfSecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Number of securities</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfSecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Number Of Securities</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_InvestmentSecuritiesBookValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Investment securities, aggregate book value</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_InvestmentSecuritiesBookValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Investment Securities Book Value</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_AggregateOfAvailableForSaleSecuritiesDebtMaturitiesAmortizedCostBasis" xlink:type="locator" xlink:label="pbhc_AggregateOfAvailableForSaleSecuritiesDebtMaturitiesAmortizedCostBasis" />
    <link:label xml:lang="en-US" xlink:label="pbhc_AggregateOfAvailableForSaleSecuritiesDebtMaturitiesAmortizedCostBasis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Amount of available-for-sale debt securities at cost, net of adjustments, which include, but are not limited to, accretion, amortization, collection of cash, previous other-than-temporary impairments (OTTI) recognized in earnings (less any cumulative-effect adjustments, as defined) and fair value hedge accounting adjustments.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_AggregateOfAvailableForSaleSecuritiesDebtMaturitiesAmortizedCostBasis" xlink:to="pbhc_AggregateOfAvailableForSaleSecuritiesDebtMaturitiesAmortizedCostBasis_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_AggregateOfHeldToMaturitySecuritiesDebtMaturitiesFairValue" xlink:type="locator" xlink:label="pbhc_AggregateOfHeldToMaturitySecuritiesDebtMaturitiesFairValue" />
    <link:label xml:lang="en-US" xlink:label="pbhc_AggregateOfHeldToMaturitySecuritiesDebtMaturitiesFairValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">This item represents the total of all debt securities grouped by maturity dates, at fair value, which are categorized as held-to-maturity.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_AggregateOfHeldToMaturitySecuritiesDebtMaturitiesFairValue" xlink:to="pbhc_AggregateOfHeldToMaturitySecuritiesDebtMaturitiesFairValue_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_AggregateOfHeldToMaturitySecuritiesDebtMaturitiesCarryingAmount" xlink:type="locator" xlink:label="pbhc_AggregateOfHeldToMaturitySecuritiesDebtMaturitiesCarryingAmount" />
    <link:label xml:lang="en-US" xlink:label="pbhc_AggregateOfHeldToMaturitySecuritiesDebtMaturitiesCarryingAmount_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Carrying amount of debt securities accounted for as held-to-maturity, net of adjustments including, but not limited to, accretion, amortization, collection of cash, previous other-than-temporary impairments (OTTI) recognized, and fair value hedge accounting adjustments.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_AggregateOfHeldToMaturitySecuritiesDebtMaturitiesCarryingAmount" xlink:to="pbhc_AggregateOfHeldToMaturitySecuritiesDebtMaturitiesCarryingAmount_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_AggregateOfAvailableForSaleSecuritiesDebtMaturitiesFairValue" xlink:type="locator" xlink:label="pbhc_AggregateOfAvailableForSaleSecuritiesDebtMaturitiesFairValue" />
    <link:label xml:lang="en-US" xlink:label="pbhc_AggregateOfAvailableForSaleSecuritiesDebtMaturitiesFairValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">This item represents the total of all debt securities grouped by maturity dates at fair value.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_AggregateOfAvailableForSaleSecuritiesDebtMaturitiesFairValue" xlink:to="pbhc_AggregateOfAvailableForSaleSecuritiesDebtMaturitiesFairValue_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AvailableForSaleSecuritiesDebtMaturitiesAmortizedCostAbstract" xlink:type="locator" xlink:label="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesAmortizedCostAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesAmortizedCostAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Available-for-sale securities, debt maturities, amortized cost [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesAmortizedCostAbstract" xlink:to="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesAmortizedCostAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesAmortizedCostAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Available For Sale Securities Debt Maturities Amortized Cost [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AvailableForSaleSecuritiesDebtMaturitiesWithinOneYearAmortizedCost" xlink:type="locator" xlink:label="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesWithinOneYearAmortizedCost" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesWithinOneYearAmortizedCost_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Due in one year or less</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesWithinOneYearAmortizedCost" xlink:to="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesWithinOneYearAmortizedCost_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesWithinOneYearAmortizedCost_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Available For Sale Securities Debt Maturities Within One Year Amortized Cost</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AvailableForSaleSecuritiesDebtMaturitiesAfterOneThroughFiveYearsAmortizedCost" xlink:type="locator" xlink:label="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesAfterOneThroughFiveYearsAmortizedCost" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesAfterOneThroughFiveYearsAmortizedCost_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Due after one year through five years</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesAfterOneThroughFiveYearsAmortizedCost" xlink:to="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesAfterOneThroughFiveYearsAmortizedCost_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesAfterOneThroughFiveYearsAmortizedCost_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Available For Sale Securities Debt Maturities After One Through Five Years Amortized Cost</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AvailableForSaleSecuritiesDebtMaturitiesAfterFiveThroughTenYearsAmortizedCost" xlink:type="locator" xlink:label="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesAfterFiveThroughTenYearsAmortizedCost" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesAfterFiveThroughTenYearsAmortizedCost_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Due after five years through ten years</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesAfterFiveThroughTenYearsAmortizedCost" xlink:to="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesAfterFiveThroughTenYearsAmortizedCost_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesAfterFiveThroughTenYearsAmortizedCost_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Available For Sale Securities Debt Maturities After Five Through Ten Years Amortized Cost</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AvailableForSaleSecuritiesDebtMaturitiesAfterTenYearsAmortizedCost" xlink:type="locator" xlink:label="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesAfterTenYearsAmortizedCost" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesAfterTenYearsAmortizedCost_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Due after ten years</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesAfterTenYearsAmortizedCost" xlink:to="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesAfterTenYearsAmortizedCost_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesAfterTenYearsAmortizedCost_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Available For Sale Securities Debt Maturities After Ten Years Amortized Cost</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AggregateOfAvailableForSaleSecuritiesDebtMaturitiesAmortizedCostBasis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Sub-total</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AggregateOfAvailableForSaleSecuritiesDebtMaturitiesAmortizedCostBasis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Aggregate Of Available For Sale Securities Debt Maturities Amortized Cost Basis</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_AvailableForSaleDebtSecuritiesAmortizedCostBasis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Amortized cost</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_AvailableForSaleSecuritiesDebtMaturitiesFairValueNet" xlink:type="locator" xlink:label="pbhc_AvailableForSaleSecuritiesDebtMaturitiesFairValueNet" />
    <link:label xml:lang="en-US" xlink:label="pbhc_AvailableForSaleSecuritiesDebtMaturitiesFairValueNet_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Available for sale securities debt maturities fair value net.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_AvailableForSaleSecuritiesDebtMaturitiesFairValueNet" xlink:to="pbhc_AvailableForSaleSecuritiesDebtMaturitiesFairValueNet_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AvailableForSaleSecuritiesDebtMaturitiesFairValueAbstract" xlink:type="locator" xlink:label="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesFairValueAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesFairValueAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Available-for-sale securities, debt maturities, Estimated Fair Value [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesFairValueAbstract" xlink:to="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesFairValueAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesFairValueAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Available For Sale Securities Debt Maturities Fair Value [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AvailableForSaleSecuritiesDebtMaturitiesWithinOneYearFairValue" xlink:type="locator" xlink:label="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesWithinOneYearFairValue" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesWithinOneYearFairValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Due in one year or less</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesWithinOneYearFairValue" xlink:to="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesWithinOneYearFairValue_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesWithinOneYearFairValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Available For Sale Securities Debt Maturities Within One Year Fair Value</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AvailableForSaleSecuritiesDebtMaturitiesAfterOneThroughFiveYearsFairValue" xlink:type="locator" xlink:label="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesAfterOneThroughFiveYearsFairValue" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesAfterOneThroughFiveYearsFairValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Due after one year through five years</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesAfterOneThroughFiveYearsFairValue" xlink:to="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesAfterOneThroughFiveYearsFairValue_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesAfterOneThroughFiveYearsFairValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Available For Sale Securities Debt Maturities After One Through Five Years Fair Value</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AvailableForSaleSecuritiesDebtMaturitiesAfterFiveThroughTenYearsFairValue" xlink:type="locator" xlink:label="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesAfterFiveThroughTenYearsFairValue" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesAfterFiveThroughTenYearsFairValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Due after five years through ten years</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesAfterFiveThroughTenYearsFairValue" xlink:to="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesAfterFiveThroughTenYearsFairValue_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesAfterFiveThroughTenYearsFairValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Available For Sale Securities Debt Maturities After Five Through Ten Years Fair Value</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AvailableForSaleSecuritiesDebtMaturitiesAfterTenYearsFairValue" xlink:type="locator" xlink:label="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesAfterTenYearsFairValue" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesAfterTenYearsFairValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Due after ten years</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesAfterTenYearsFairValue" xlink:to="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesAfterTenYearsFairValue_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AvailableForSaleSecuritiesDebtMaturitiesAfterTenYearsFairValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Available For Sale Securities Debt Maturities After Ten Years Fair Value</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AggregateOfAvailableForSaleSecuritiesDebtMaturitiesFairValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Sub-total</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AggregateOfAvailableForSaleSecuritiesDebtMaturitiesFairValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Aggregate Of Available For Sale Securities Debt Maturities Fair Value</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AvailableForSaleSecuritiesDebtMaturitiesFairValueNet_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Available-for-sale securities, debt maturities, fair value, totals</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AvailableForSaleSecuritiesDebtMaturitiesFairValueNet_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Available For Sale Securities Debt Maturities Fair Value Net</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_HeldToMaturitySecuritiesDebtMaturitiesNetCarryingAmountAbstract" xlink:type="locator" xlink:label="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesNetCarryingAmountAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesNetCarryingAmountAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Held-to-maturity Securities, debt maturities, amortized cost [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesNetCarryingAmountAbstract" xlink:to="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesNetCarryingAmountAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesNetCarryingAmountAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Held To Maturity Securities Debt Maturities Net Carrying Amount [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_HeldToMaturitySecuritiesDebtMaturitiesWithinOneYearNetCarryingAmount" xlink:type="locator" xlink:label="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesWithinOneYearNetCarryingAmount" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesWithinOneYearNetCarryingAmount_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Due in one year or less</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesWithinOneYearNetCarryingAmount" xlink:to="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesWithinOneYearNetCarryingAmount_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesWithinOneYearNetCarryingAmount_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Held To Maturity Securities Debt Maturities Within One Year Net Carrying Amount</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_HeldToMaturitySecuritiesDebtMaturitiesAfterOneThroughFiveYearsNetCarryingAmount" xlink:type="locator" xlink:label="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesAfterOneThroughFiveYearsNetCarryingAmount" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesAfterOneThroughFiveYearsNetCarryingAmount_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Due after one year through five years</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesAfterOneThroughFiveYearsNetCarryingAmount" xlink:to="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesAfterOneThroughFiveYearsNetCarryingAmount_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesAfterOneThroughFiveYearsNetCarryingAmount_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Held To Maturity Securities Debt Maturities After One Through Five Years Net Carrying Amount</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_HeldToMaturitySecuritiesDebtMaturitiesAfterFiveThroughTenYearsNetCarryingAmount" xlink:type="locator" xlink:label="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesAfterFiveThroughTenYearsNetCarryingAmount" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesAfterFiveThroughTenYearsNetCarryingAmount_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Due after five years through ten years</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesAfterFiveThroughTenYearsNetCarryingAmount" xlink:to="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesAfterFiveThroughTenYearsNetCarryingAmount_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesAfterFiveThroughTenYearsNetCarryingAmount_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Held To Maturity Securities Debt Maturities After Five Through Ten Years Net Carrying Amount</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_HeldToMaturitySecuritiesDebtMaturitiesAfterTenYearsNetCarryingAmount" xlink:type="locator" xlink:label="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesAfterTenYearsNetCarryingAmount" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesAfterTenYearsNetCarryingAmount_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Due after ten years</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesAfterTenYearsNetCarryingAmount" xlink:to="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesAfterTenYearsNetCarryingAmount_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesAfterTenYearsNetCarryingAmount_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Held To Maturity Securities Debt Maturities After Ten Years Net Carrying Amount</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AggregateOfHeldToMaturitySecuritiesDebtMaturitiesCarryingAmount_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Sub-total</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AggregateOfHeldToMaturitySecuritiesDebtMaturitiesCarryingAmount_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Aggregate Of Held To Maturity Securities Debt Maturities Carrying Amount</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Held-to-maturity securities, debt maturities, Amortized Cost</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_HeldToMaturitySecuritiesDebtMaturitiesFairValueAbstract" xlink:type="locator" xlink:label="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesFairValueAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesFairValueAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Held-to-maturity Securities, debt maturities, Estimated Fair Value [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesFairValueAbstract" xlink:to="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesFairValueAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesFairValueAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Held To Maturity Securities Debt Maturities Fair Value [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_HeldToMaturitySecuritiesDebtMaturitiesWithinOneYearFairValue" xlink:type="locator" xlink:label="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesWithinOneYearFairValue" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesWithinOneYearFairValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Due in one year or less</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesWithinOneYearFairValue" xlink:to="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesWithinOneYearFairValue_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesWithinOneYearFairValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Held To Maturity Securities Debt Maturities Within One Year Fair Value</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_HeldToMaturitySecuritiesDebtMaturitiesAfterOneThroughFiveYearsFairValue" xlink:type="locator" xlink:label="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesAfterOneThroughFiveYearsFairValue" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesAfterOneThroughFiveYearsFairValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Due after one year through five years</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesAfterOneThroughFiveYearsFairValue" xlink:to="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesAfterOneThroughFiveYearsFairValue_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesAfterOneThroughFiveYearsFairValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Held To Maturity Securities Debt Maturities After One Through Five Years Fair Value</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_HeldToMaturitySecuritiesDebtMaturitiesAfterFiveThroughTenYearsFairValue" xlink:type="locator" xlink:label="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesAfterFiveThroughTenYearsFairValue" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesAfterFiveThroughTenYearsFairValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Due after five years through ten years</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesAfterFiveThroughTenYearsFairValue" xlink:to="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesAfterFiveThroughTenYearsFairValue_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesAfterFiveThroughTenYearsFairValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Held To Maturity Securities Debt Maturities After Five Through Ten Years Fair Value</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_HeldToMaturitySecuritiesDebtMaturitiesAfterTenYearsFairValue" xlink:type="locator" xlink:label="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesAfterTenYearsFairValue" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesAfterTenYearsFairValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Due after ten years</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesAfterTenYearsFairValue" xlink:to="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesAfterTenYearsFairValue_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesDebtMaturitiesAfterTenYearsFairValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Held To Maturity Securities Debt Maturities After Ten Years Fair Value</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AggregateOfHeldToMaturitySecuritiesDebtMaturitiesFairValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Sub-total</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AggregateOfHeldToMaturitySecuritiesDebtMaturitiesFairValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Aggregate Of Held To Maturity Securities Debt Maturities Fair Value</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_HeldToMaturityNumberOfSecuritiesInUnrealizedLossPositions" xlink:type="locator" xlink:label="pbhc_HeldToMaturityNumberOfSecuritiesInUnrealizedLossPositions" />
    <link:label xml:lang="en-US" xlink:label="pbhc_HeldToMaturityNumberOfSecuritiesInUnrealizedLossPositions_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">This element represent the number of held to maturity securities in unrealized loss positions.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_HeldToMaturityNumberOfSecuritiesInUnrealizedLossPositions" xlink:to="pbhc_HeldToMaturityNumberOfSecuritiesInUnrealizedLossPositions_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DebtSecuritiesMember" xlink:type="locator" xlink:label="us-gaap_DebtSecuritiesMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtSecuritiesMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Debt Securities [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DebtSecuritiesMember" xlink:to="us-gaap_DebtSecuritiesMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtSecuritiesMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Debt Securities [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPositionLessThan12MonthsNumberOfPositions" xlink:type="locator" xlink:label="us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPositionLessThan12MonthsNumberOfPositions" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPositionLessThan12MonthsNumberOfPositions_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Number of securities in unrealized loss positions, less than twelve months</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPositionLessThan12MonthsNumberOfPositions" xlink:to="us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPositionLessThan12MonthsNumberOfPositions_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPositionLessThan12MonthsNumberOfPositions_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Debt Securities Available For Sale Continuous Unrealized Loss Position Less Than12 Months Number Of Positions</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPosition12MonthsOrLongerNumberOfPositions" xlink:type="locator" xlink:label="us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPosition12MonthsOrLongerNumberOfPositions" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPosition12MonthsOrLongerNumberOfPositions_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Number of securities in unrealized loss positions, twelve months or more</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPosition12MonthsOrLongerNumberOfPositions" xlink:to="us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPosition12MonthsOrLongerNumberOfPositions_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPosition12MonthsOrLongerNumberOfPositions_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Debt Securities Available For Sale Continuous Unrealized Loss Position12 Months Or Longer Number Of Positions</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DebtSecuritiesAvailableForSaleUnrealizedLossPositionNumberOfPositions" xlink:type="locator" xlink:label="us-gaap_DebtSecuritiesAvailableForSaleUnrealizedLossPositionNumberOfPositions" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtSecuritiesAvailableForSaleUnrealizedLossPositionNumberOfPositions_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Number of securities in unrealized loss positions</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DebtSecuritiesAvailableForSaleUnrealizedLossPositionNumberOfPositions" xlink:to="us-gaap_DebtSecuritiesAvailableForSaleUnrealizedLossPositionNumberOfPositions_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtSecuritiesAvailableForSaleUnrealizedLossPositionNumberOfPositions_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Debt Securities Available For Sale Unrealized Loss Position Number Of Positions</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AvailableForSaleSecuritiesContinuousUnrealizedLossPositionAccumulatedLossAbstract" xlink:type="locator" xlink:label="us-gaap_AvailableForSaleSecuritiesContinuousUnrealizedLossPositionAccumulatedLossAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AvailableForSaleSecuritiesContinuousUnrealizedLossPositionAccumulatedLossAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Available-for-sale Securities, Continuous Unrealized Loss Position, Accumulated Loss [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AvailableForSaleSecuritiesContinuousUnrealizedLossPositionAccumulatedLossAbstract" xlink:to="us-gaap_AvailableForSaleSecuritiesContinuousUnrealizedLossPositionAccumulatedLossAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AvailableForSaleSecuritiesContinuousUnrealizedLossPositionAccumulatedLossAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Available For Sale Securities Continuous Unrealized Loss Position Accumulated Loss [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPositionLessThan12MonthsAccumulatedLoss" xlink:type="locator" xlink:label="us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPositionLessThan12MonthsAccumulatedLoss" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPositionLessThan12MonthsAccumulatedLoss_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Less than twelve months Unrealized Losses</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPositionLessThan12MonthsAccumulatedLoss" xlink:to="us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPositionLessThan12MonthsAccumulatedLoss_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPositionLessThan12MonthsAccumulatedLoss_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Debt Securities Available For Sale Continuous Unrealized Loss Position Less Than12 Months Accumulated Loss</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPositionLessThan12Months" xlink:type="locator" xlink:label="us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPositionLessThan12Months" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPositionLessThan12Months_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Less than twelve months Fair Value</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPositionLessThan12Months" xlink:to="us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPositionLessThan12Months_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPositionLessThan12Months_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Debt Securities Available For Sale Continuous Unrealized Loss Position Less Than12 Months</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPosition12MonthsOrLongerAccumulatedLoss" xlink:type="locator" xlink:label="us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPosition12MonthsOrLongerAccumulatedLoss" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPosition12MonthsOrLongerAccumulatedLoss_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Twelve months or more Unrealized Losses</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPosition12MonthsOrLongerAccumulatedLoss" xlink:to="us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPosition12MonthsOrLongerAccumulatedLoss_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPosition12MonthsOrLongerAccumulatedLoss_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Debt Securities Available For Sale Continuous Unrealized Loss Position12 Months Or Longer Accumulated Loss</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPosition12MonthsOrLonger" xlink:type="locator" xlink:label="us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPosition12MonthsOrLonger" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPosition12MonthsOrLonger_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Twelve months or more Fair Value</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPosition12MonthsOrLonger" xlink:to="us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPosition12MonthsOrLonger_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPosition12MonthsOrLonger_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Debt Securities Available For Sale Continuous Unrealized Loss Position12 Months Or Longer</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DebtSecuritiesAvailableForSaleUnrealizedLossPositionAccumulatedLoss" xlink:type="locator" xlink:label="us-gaap_DebtSecuritiesAvailableForSaleUnrealizedLossPositionAccumulatedLoss" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtSecuritiesAvailableForSaleUnrealizedLossPositionAccumulatedLoss_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedTotalLabel">Total Unrealized Losses</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DebtSecuritiesAvailableForSaleUnrealizedLossPositionAccumulatedLoss" xlink:to="us-gaap_DebtSecuritiesAvailableForSaleUnrealizedLossPositionAccumulatedLoss_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtSecuritiesAvailableForSaleUnrealizedLossPositionAccumulatedLoss_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Debt Securities Available For Sale Unrealized Loss Position Accumulated Loss</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DebtSecuritiesAvailableForSaleUnrealizedLossPosition" xlink:type="locator" xlink:label="us-gaap_DebtSecuritiesAvailableForSaleUnrealizedLossPosition" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtSecuritiesAvailableForSaleUnrealizedLossPosition_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Total Fair Value</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DebtSecuritiesAvailableForSaleUnrealizedLossPosition" xlink:to="us-gaap_DebtSecuritiesAvailableForSaleUnrealizedLossPosition_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtSecuritiesAvailableForSaleUnrealizedLossPosition_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Debt Securities Available For Sale Unrealized Loss Position</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionFairValueAbstract" xlink:type="locator" xlink:label="us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionFairValueAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionFairValueAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Held-to-maturity Securities, Continuous Unrealized Loss Position, Fair Value [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionFairValueAbstract" xlink:to="us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionFairValueAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionFairValueAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Held To Maturity Securities Continuous Unrealized Loss Position Fair Value [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_HeldToMaturityNumberOfSecuritiesInUnrealizedLossPositionsLessThanTwelveMonths_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Number of securities in unrealized loss positions, less than twelve months</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_HeldToMaturityNumberOfSecuritiesInUnrealizedLossPositionsLessThanTwelveMonths_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Held To Maturity Number Of Securities In Unrealized Loss Positions Less Than Twelve Months</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_HeldToMaturityNumberOfSecuritiesInUnrealizedLossPositionsTwelveMonthsOrMore_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Number of securities in unrealized loss positions, twelve months or more</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_HeldToMaturityNumberOfSecuritiesInUnrealizedLossPositionsTwelveMonthsOrMore_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Held To Maturity Number Of Securities In Unrealized Loss Positions Twelve Months Or More</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_HeldToMaturityNumberOfSecuritiesInUnrealizedLossPositions_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Number of securities in unrealized loss positions</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_HeldToMaturityNumberOfSecuritiesInUnrealizedLossPositions_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Held To Maturity Number Of Securities In Unrealized Loss Positions</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionAccumulatedLossAbstract" xlink:type="locator" xlink:label="us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionAccumulatedLossAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionAccumulatedLossAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Held-to-maturity Securities, Continuous Unrealized Loss Position, Accumulated Loss [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionAccumulatedLossAbstract" xlink:to="us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionAccumulatedLossAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionAccumulatedLossAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Held To Maturity Securities Continuous Unrealized Loss Position Accumulated Loss [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionLessThan12MonthsAccumulatedLoss" xlink:type="locator" xlink:label="us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionLessThan12MonthsAccumulatedLoss" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionLessThan12MonthsAccumulatedLoss_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Less than twelve months Unrealized Losses</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionLessThan12MonthsAccumulatedLoss" xlink:to="us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionLessThan12MonthsAccumulatedLoss_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionLessThan12MonthsAccumulatedLoss_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Held To Maturity Securities Continuous Unrealized Loss Position Less Than12 Months Accumulated Loss</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionLessThanTwelveMonthsFairValue" xlink:type="locator" xlink:label="us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionLessThanTwelveMonthsFairValue" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionLessThanTwelveMonthsFairValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Less than twelve months Fair Value</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionLessThanTwelveMonthsFairValue" xlink:to="us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionLessThanTwelveMonthsFairValue_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionLessThanTwelveMonthsFairValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Held To Maturity Securities Continuous Unrealized Loss Position Less Than Twelve Months Fair Value</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPosition12MonthsOrLongerAccumulatedLoss" xlink:type="locator" xlink:label="us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPosition12MonthsOrLongerAccumulatedLoss" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPosition12MonthsOrLongerAccumulatedLoss_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Twelve months or more Unrealized Losses</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPosition12MonthsOrLongerAccumulatedLoss" xlink:to="us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPosition12MonthsOrLongerAccumulatedLoss_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPosition12MonthsOrLongerAccumulatedLoss_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Held To Maturity Securities Continuous Unrealized Loss Position12 Months Or Longer Accumulated Loss</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionTwelveMonthsOrLongerFairValue" xlink:type="locator" xlink:label="us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionTwelveMonthsOrLongerFairValue" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionTwelveMonthsOrLongerFairValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Twelve months or more Fair Value</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionTwelveMonthsOrLongerFairValue" xlink:to="us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionTwelveMonthsOrLongerFairValue_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionTwelveMonthsOrLongerFairValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Held To Maturity Securities Continuous Unrealized Loss Position Twelve Months Or Longer Fair Value</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionAccumulatedLoss" xlink:type="locator" xlink:label="us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionAccumulatedLoss" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionAccumulatedLoss_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedTotalLabel">Total Unrealized Losses</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionAccumulatedLoss" xlink:to="us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionAccumulatedLoss_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionAccumulatedLoss_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Held To Maturity Securities Continuous Unrealized Loss Position Accumulated Loss</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionFairValue" xlink:type="locator" xlink:label="us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionFairValue" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionFairValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Total Fair Value</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionFairValue" xlink:to="us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionFairValue_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionFairValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Held To Maturity Securities Continuous Unrealized Loss Position Fair Value</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_AvailableForSaleSecuritiesContinuousUnrealizedLossPositionTwelveMonthsOrMoreAmortizedHistoricalCostBasis" xlink:type="locator" xlink:label="pbhc_AvailableForSaleSecuritiesContinuousUnrealizedLossPositionTwelveMonthsOrMoreAmortizedHistoricalCostBasis" />
    <link:label xml:lang="en-US" xlink:label="pbhc_AvailableForSaleSecuritiesContinuousUnrealizedLossPositionTwelveMonthsOrMoreAmortizedHistoricalCostBasis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Available for sale securities continuous unrealized loss position twelve months or more amortized historical cost basis.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_AvailableForSaleSecuritiesContinuousUnrealizedLossPositionTwelveMonthsOrMoreAmortizedHistoricalCostBasis" xlink:to="pbhc_AvailableForSaleSecuritiesContinuousUnrealizedLossPositionTwelveMonthsOrMoreAmortizedHistoricalCostBasis_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_NumberOfSecuritiesHeldToMaturity" xlink:type="locator" xlink:label="pbhc_NumberOfSecuritiesHeldToMaturity" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfSecuritiesHeldToMaturity_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Number of securities held to maturity.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_NumberOfSecuritiesHeldToMaturity" xlink:to="pbhc_NumberOfSecuritiesHeldToMaturity_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionGreaterThanTwelveMonthsAmortizedHistoricalCostBasis" xlink:type="locator" xlink:label="pbhc_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionGreaterThanTwelveMonthsAmortizedHistoricalCostBasis" />
    <link:label xml:lang="en-US" xlink:label="pbhc_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionGreaterThanTwelveMonthsAmortizedHistoricalCostBasis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Held to maturity securities continuous unrealized loss position greater than twelve months amortized historical cost basis.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionGreaterThanTwelveMonthsAmortizedHistoricalCostBasis" xlink:to="pbhc_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionGreaterThanTwelveMonthsAmortizedHistoricalCostBasis_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_AvailableForSaleSecuritiesOfAmortizedCostBasisAbstract" xlink:type="locator" xlink:label="pbhc_AvailableForSaleSecuritiesOfAmortizedCostBasisAbstract" />
    <link:label xml:lang="en-US" xlink:label="pbhc_AvailableForSaleSecuritiesOfAmortizedCostBasisAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Available for sale securities of amortized cost basis.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_AvailableForSaleSecuritiesOfAmortizedCostBasisAbstract" xlink:to="pbhc_AvailableForSaleSecuritiesOfAmortizedCostBasisAbstract_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_AvailableForSaleSecuritiesContinuousUnrealizedLossPositionTwelveMonthsOrLongerAccumulatedLossPercentage" xlink:type="locator" xlink:label="pbhc_AvailableForSaleSecuritiesContinuousUnrealizedLossPositionTwelveMonthsOrLongerAccumulatedLossPercentage" />
    <link:label xml:lang="en-US" xlink:label="pbhc_AvailableForSaleSecuritiesContinuousUnrealizedLossPositionTwelveMonthsOrLongerAccumulatedLossPercentage_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Available for sale securities continuous unrealized loss position twelve months or longer accumulated loss percentage.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_AvailableForSaleSecuritiesContinuousUnrealizedLossPositionTwelveMonthsOrLongerAccumulatedLossPercentage" xlink:to="pbhc_AvailableForSaleSecuritiesContinuousUnrealizedLossPositionTwelveMonthsOrLongerAccumulatedLossPercentage_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionTwelveMonthsOrLongerAccumulatedLossPercentage" xlink:type="locator" xlink:label="pbhc_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionTwelveMonthsOrLongerAccumulatedLossPercentage" />
    <link:label xml:lang="en-US" xlink:label="pbhc_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionTwelveMonthsOrLongerAccumulatedLossPercentage_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Held to maturity securities continuous unrealized loss position twelve months or longer accumulated loss percentage.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionTwelveMonthsOrLongerAccumulatedLossPercentage" xlink:to="pbhc_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionTwelveMonthsOrLongerAccumulatedLossPercentage_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_AvailableForSaleSecuritiesOfAmortizedCostBasisAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Available-for-sale Securities, Amortized Cost Basis [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AvailableForSaleSecuritiesOfAmortizedCostBasisAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Available For Sale Securities Of Amortized Cost Basis [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AvailableForSaleSecuritiesContinuousUnrealizedLossPositionTwelveMonthsOrMoreAmortizedHistoricalCostBasis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Available for sale securities continuous unrealized loss position twelve months or more amortized historical cost basis</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AvailableForSaleSecuritiesContinuousUnrealizedLossPositionTwelveMonthsOrMoreAmortizedHistoricalCostBasis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Available For Sale Securities Continuous Unrealized Loss Position Twelve Months Or More Amortized Historical Cost Basis</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPosition12MonthsOrLongerAccumulatedLoss_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Available for sale securities unrealized aggregate loss</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AvailableForSaleSecuritiesContinuousUnrealizedLossPositionTwelveMonthsOrLongerAccumulatedLossPercentage_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Available for sale securities unrealized aggregate loss percentage</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AvailableForSaleSecuritiesContinuousUnrealizedLossPositionTwelveMonthsOrLongerAccumulatedLossPercentage_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Available For Sale Securities Continuous Unrealized Loss Position Twelve Months Or Longer Accumulated Loss Percentage</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_HeldToMaturitySecuritiesAbstract" xlink:type="locator" xlink:label="us-gaap_HeldToMaturitySecuritiesAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Held-to-maturity Securities [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_HeldToMaturitySecuritiesAbstract" xlink:to="us-gaap_HeldToMaturitySecuritiesAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Held To Maturity Securities [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfSecuritiesHeldToMaturity_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Held to maturity securities</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfSecuritiesHeldToMaturity_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Number Of Securities Held To Maturity</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionGreaterThanTwelveMonthsAmortizedHistoricalCostBasis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Held to maturity securities continuous unrealized loss position greater than twelve months amortized historical cost basis</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionGreaterThanTwelveMonthsAmortizedHistoricalCostBasis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Held To Maturity Securities Continuous Unrealized Loss Position Greater Than Twelve Months Amortized Historical Cost Basis</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPosition12MonthsOrLongerAccumulatedLoss_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Held-to-maturity securities aggregate unrealized loss</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionTwelveMonthsOrLongerAccumulatedLossPercentage_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Held To Maturity Securities Continuous Unrealized Loss Position Twelve Months Or Longer Accumulated Loss Percentage</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionTwelveMonthsOrLongerAccumulatedLossPercentage_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Held To Maturity Securities Continuous Unrealized Loss Position Twelve Months Or Longer Accumulated Loss Percentage</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_MonthsOfUnrealizedLossPositions" xlink:type="locator" xlink:label="pbhc_MonthsOfUnrealizedLossPositions" />
    <link:label xml:lang="en-US" xlink:label="pbhc_MonthsOfUnrealizedLossPositions_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Number of months of unrealized loss position.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_MonthsOfUnrealizedLossPositions" xlink:to="pbhc_MonthsOfUnrealizedLossPositions_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_TotalNumberOfSecuritiesContinuousUnrealizedLossPositionForTwelveMonthsOrMore" xlink:type="locator" xlink:label="pbhc_TotalNumberOfSecuritiesContinuousUnrealizedLossPositionForTwelveMonthsOrMore" />
    <link:label xml:lang="en-US" xlink:label="pbhc_TotalNumberOfSecuritiesContinuousUnrealizedLossPositionForTwelveMonthsOrMore_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Total number of securities continuous unrealized loss position for twelve months or more.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_TotalNumberOfSecuritiesContinuousUnrealizedLossPositionForTwelveMonthsOrMore" xlink:to="pbhc_TotalNumberOfSecuritiesContinuousUnrealizedLossPositionForTwelveMonthsOrMore_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_NumberOfSecuritiesAmortized" xlink:type="locator" xlink:label="pbhc_NumberOfSecuritiesAmortized" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfSecuritiesAmortized_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Number of securities amortized.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_NumberOfSecuritiesAmortized" xlink:to="pbhc_NumberOfSecuritiesAmortized_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_PercentageOfSecuritiesAmortizedOfTotalSecurities" xlink:type="locator" xlink:label="pbhc_PercentageOfSecuritiesAmortizedOfTotalSecurities" />
    <link:label xml:lang="en-US" xlink:label="pbhc_PercentageOfSecuritiesAmortizedOfTotalSecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Percentage of securities amortized of total securities.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_PercentageOfSecuritiesAmortizedOfTotalSecurities" xlink:to="pbhc_PercentageOfSecuritiesAmortizedOfTotalSecurities_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_MonthsOfUnrealizedLossPositions_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Months of unrealized loss positions</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MonthsOfUnrealizedLossPositions_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Months Of Unrealized Loss Positions</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_TotalNumberOfSecuritiesContinuousUnrealizedLossPositionForTwelveMonthsOrMore_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Number of securities in unrealized loss positions, twelve months or more</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_TotalNumberOfSecuritiesContinuousUnrealizedLossPositionForTwelveMonthsOrMore_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Total Number Of Securities Continuous Unrealized Loss Position For Twelve Months Or More</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfSecuritiesAmortized_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Number of securities amortized</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfSecuritiesAmortized_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Number Of Securities Amortized</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_PercentageOfSecuritiesAmortizedOfTotalSecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Percentage of securities amortized</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_PercentageOfSecuritiesAmortizedOfTotalSecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Percentage Of Securities Amortized Of Total Securities</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPosition12MonthsOrLonger_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Available for sale securities aggregate market value</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_DebtSecuritiesAggregateAmortizedHistoricalCost" xlink:type="locator" xlink:label="pbhc_DebtSecuritiesAggregateAmortizedHistoricalCost" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DebtSecuritiesAggregateAmortizedHistoricalCost_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Debt securities aggregate amortized historical cost.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_DebtSecuritiesAggregateAmortizedHistoricalCost" xlink:to="pbhc_DebtSecuritiesAggregateAmortizedHistoricalCost_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DebtSecuritiesAggregateAmortizedHistoricalCost_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Available for sale securities continuous unrealized loss position twelve months or more amortized historical cost basis</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DebtSecuritiesAggregateAmortizedHistoricalCost_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Debt Securities Aggregate Amortized Historical Cost</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesContinuousUnrealizedLossPositionTwelveMonthsOrLongerFairValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Held to maturity sale securities aggregate fair value</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ProceedsFromSaleOfAvailableForSaleSecuritiesAndHeldToMaturitySecurities" xlink:type="locator" xlink:label="pbhc_ProceedsFromSaleOfAvailableForSaleSecuritiesAndHeldToMaturitySecurities" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ProceedsFromSaleOfAvailableForSaleSecuritiesAndHeldToMaturitySecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The cash inflow associated with the sale of debt and equity securities classified as available-for-sale securities and sale of securities that had been designated as held-to-maturity.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ProceedsFromSaleOfAvailableForSaleSecuritiesAndHeldToMaturitySecurities" xlink:to="pbhc_ProceedsFromSaleOfAvailableForSaleSecuritiesAndHeldToMaturitySecurities_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ProceedsFromSaleOfAvailableForSaleSecuritiesAndHeldToMaturitySecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Proceeds on sales and redemptions of securities</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ProceedsFromSaleOfAvailableForSaleSecuritiesAndHeldToMaturitySecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Proceeds From Sale Of Available For Sale Securities And Held To Maturity Securities</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_RealizedGainsOnInvestmentsSold" xlink:type="locator" xlink:label="pbhc_RealizedGainsOnInvestmentsSold" />
    <link:label xml:lang="en-US" xlink:label="pbhc_RealizedGainsOnInvestmentsSold_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The net realized gain on investments sold during the period, not including gains (losses) on securities separately or otherwise categorized as trading, available-for-sale, or held-to-maturity, which, for cash flow reporting, is a component of proceeds from investing activities.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_RealizedGainsOnInvestmentsSold" xlink:to="pbhc_RealizedGainsOnInvestmentsSold_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_RealizedLossesOnSaleOfInvestments" xlink:type="locator" xlink:label="pbhc_RealizedLossesOnSaleOfInvestments" />
    <link:label xml:lang="en-US" xlink:label="pbhc_RealizedLossesOnSaleOfInvestments_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The net realized loss on investments sold during the period, not including gains (losses) on securities separately or otherwise categorized as trading, available-for-sale, or held-to-maturity, which, for cash flow reporting, is a component of proceeds from investing activities.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_RealizedLossesOnSaleOfInvestments" xlink:to="pbhc_RealizedLossesOnSaleOfInvestments_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_GainLossOnSaleOfInvestmentsExcludingRealizedGainOnMarketableEquitySecurity" xlink:type="locator" xlink:label="pbhc_GainLossOnSaleOfInvestmentsExcludingRealizedGainOnMarketableEquitySecurity" />
    <link:label xml:lang="en-US" xlink:label="pbhc_GainLossOnSaleOfInvestmentsExcludingRealizedGainOnMarketableEquitySecurity_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Gain loss on sale of investments excluding realized gain on marketable equity security.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_GainLossOnSaleOfInvestmentsExcludingRealizedGainOnMarketableEquitySecurity" xlink:to="pbhc_GainLossOnSaleOfInvestmentsExcludingRealizedGainOnMarketableEquitySecurity_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_GainLossOnSaleOfInvestmentsAbstract" xlink:type="locator" xlink:label="us-gaap_GainLossOnSaleOfInvestmentsAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_GainLossOnSaleOfInvestmentsAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Gain Loss On Sale Of Investments [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_GainLossOnSaleOfInvestmentsAbstract" xlink:to="us-gaap_GainLossOnSaleOfInvestmentsAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_RealizedGainsOnInvestmentsSold_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Realized gains on investments</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_RealizedGainsOnInvestmentsSold_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Realized Gains On Investments Sold</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_RealizedLossesOnSaleOfInvestments_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Realized losses on investments</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_RealizedLossesOnSaleOfInvestments_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Realized Losses On Sale Of Investments</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_GainLossOnSaleOfInvestmentsExcludingRealizedGainOnMarketableEquitySecurity_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Total</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_GainLossOnSaleOfInvestmentsExcludingRealizedGainOnMarketableEquitySecurity_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Gain Loss On Sale Of Investments Excluding Realized Gain On Marketable Equity Security</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_SecuritiesPledgedToCollateralizeDeposit" xlink:type="locator" xlink:label="pbhc_SecuritiesPledgedToCollateralizeDeposit" />
    <link:label xml:lang="en-US" xlink:label="pbhc_SecuritiesPledgedToCollateralizeDeposit_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">This element represent the fair value of securities pledged to collateralize municipal deposits.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_SecuritiesPledgedToCollateralizeDeposit" xlink:to="pbhc_SecuritiesPledgedToCollateralizeDeposit_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_SecuritiesPledgedToCollateralizeBorrowing" xlink:type="locator" xlink:label="pbhc_SecuritiesPledgedToCollateralizeBorrowing" />
    <link:label xml:lang="en-US" xlink:label="pbhc_SecuritiesPledgedToCollateralizeBorrowing_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">This element represent the fair value of securities pledged to collateralize borrowings.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_SecuritiesPledgedToCollateralizeBorrowing" xlink:to="pbhc_SecuritiesPledgedToCollateralizeBorrowing_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_GainLossOnSaleOfInvestmentsAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Gain (Loss) on Sale of Investments [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_SecuritiesPledgedToCollateralizeDeposit_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Securities pledged to collateralize deposit</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_SecuritiesPledgedToCollateralizeDeposit_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Securities Pledged To Collateralize Deposit</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_SecuritiesPledgedToCollateralizeBorrowing_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Securities pledged to collateralize borrowing</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_SecuritiesPledgedToCollateralizeBorrowing_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Securities Pledged To Collateralize Borrowing</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ScheduleOfAccountsNotesLoansAndFinancingReceivableTable" xlink:type="locator" xlink:label="us-gaap_ScheduleOfAccountsNotesLoansAndFinancingReceivableTable" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfAccountsNotesLoansAndFinancingReceivableTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Schedule Of Accounts Notes Loans And Financing Receivable [Table]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ScheduleOfAccountsNotesLoansAndFinancingReceivableTable" xlink:to="us-gaap_ScheduleOfAccountsNotesLoansAndFinancingReceivableTable_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfAccountsNotesLoansAndFinancingReceivableTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Accounts Notes Loans And Financing Receivable [Table]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FinancingReceivablePortfolioSegmentAxis" xlink:type="locator" xlink:label="us-gaap_FinancingReceivablePortfolioSegmentAxis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivablePortfolioSegmentAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Financing Receivable Portfolio Segment</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FinancingReceivablePortfolioSegmentAxis" xlink:to="us-gaap_FinancingReceivablePortfolioSegmentAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivablePortfolioSegmentAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Financing Receivable Portfolio Segment [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FinancingReceivablePortfolioSegmentDomain" xlink:type="locator" xlink:label="us-gaap_FinancingReceivablePortfolioSegmentDomain" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivablePortfolioSegmentDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Financing Receivable Portfolio Segment</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FinancingReceivablePortfolioSegmentDomain" xlink:to="us-gaap_FinancingReceivablePortfolioSegmentDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivablePortfolioSegmentDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Financing Receivable Portfolio Segment [Domain]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ResidentialPortfolioSegmentMember" xlink:type="locator" xlink:label="us-gaap_ResidentialPortfolioSegmentMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ResidentialPortfolioSegmentMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Residential Mortgage Loans [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ResidentialPortfolioSegmentMember" xlink:to="us-gaap_ResidentialPortfolioSegmentMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ResidentialPortfolioSegmentMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Residential Portfolio Segment [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_CommercialPortfolioSegmentMember" xlink:type="locator" xlink:label="us-gaap_CommercialPortfolioSegmentMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CommercialPortfolioSegmentMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Commercial Loans [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_CommercialPortfolioSegmentMember" xlink:to="us-gaap_CommercialPortfolioSegmentMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CommercialPortfolioSegmentMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Commercial Portfolio Segment [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ConsumerPortfolioSegmentMember" xlink:type="locator" xlink:label="us-gaap_ConsumerPortfolioSegmentMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ConsumerPortfolioSegmentMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Consumer Loans [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ConsumerPortfolioSegmentMember" xlink:to="us-gaap_ConsumerPortfolioSegmentMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ConsumerPortfolioSegmentMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Consumer Portfolio Segment [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FinancingReceivableRecordedInvestmentByClassOfFinancingReceivableAxis" xlink:type="locator" xlink:label="us-gaap_FinancingReceivableRecordedInvestmentByClassOfFinancingReceivableAxis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableRecordedInvestmentByClassOfFinancingReceivableAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Class of Financing Receivable</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FinancingReceivableRecordedInvestmentByClassOfFinancingReceivableAxis" xlink:to="us-gaap_FinancingReceivableRecordedInvestmentByClassOfFinancingReceivableAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableRecordedInvestmentByClassOfFinancingReceivableAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Financing Receivable Recorded Investment By Class Of Financing Receivable [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FinancingReceivableRecordedInvestmentClassOfFinancingReceivableDomain" xlink:type="locator" xlink:label="us-gaap_FinancingReceivableRecordedInvestmentClassOfFinancingReceivableDomain" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableRecordedInvestmentClassOfFinancingReceivableDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Class of Financing Receivable</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FinancingReceivableRecordedInvestmentClassOfFinancingReceivableDomain" xlink:to="us-gaap_FinancingReceivableRecordedInvestmentClassOfFinancingReceivableDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableRecordedInvestmentClassOfFinancingReceivableDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Financing Receivable Recorded Investment Class Of Financing Receivable [Domain]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_FamilyFirstLienResidentialMortgages1To4ExcludingLoansHeldForSaleMember" xlink:type="locator" xlink:label="pbhc_FamilyFirstLienResidentialMortgages1To4ExcludingLoansHeldForSaleMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_FamilyFirstLienResidentialMortgages1To4ExcludingLoansHeldForSaleMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Family first lien residential mortgages 1 To 4 excluding loans held for sale.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_FamilyFirstLienResidentialMortgages1To4ExcludingLoansHeldForSaleMember" xlink:to="pbhc_FamilyFirstLienResidentialMortgages1To4ExcludingLoansHeldForSaleMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_FamilyFirstLienResidentialMortgages1To4ExcludingLoansHeldForSaleMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">1-4 Family First Lien Residential Mortgages [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_FamilyFirstLienResidentialMortgages1To4ExcludingLoansHeldForSaleMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Family First Lien Residential Mortgages1 To4 Excluding Loans Held For Sale [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ConstructionLoansMember" xlink:type="locator" xlink:label="us-gaap_ConstructionLoansMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ConstructionLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Construction [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ConstructionLoansMember" xlink:to="us-gaap_ConstructionLoansMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ConstructionLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Construction Loans [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_LoansHeldForSaleMember" xlink:type="locator" xlink:label="pbhc_LoansHeldForSaleMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_LoansHeldForSaleMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Loans held-for-sale.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_LoansHeldForSaleMember" xlink:to="pbhc_LoansHeldForSaleMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_LoansHeldForSaleMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Loans Held-For-Sale [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_LoansHeldForSaleMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Loans Held For Sale [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_RealEstateLoanMember" xlink:type="locator" xlink:label="us-gaap_RealEstateLoanMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RealEstateLoanMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Real Estate [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_RealEstateLoanMember" xlink:to="us-gaap_RealEstateLoanMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RealEstateLoanMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Real Estate Loan [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_LinesOfCreditMember" xlink:type="locator" xlink:label="pbhc_LinesOfCreditMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_LinesOfCreditMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Class of financing receivables related to lines of credit receivables.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_LinesOfCreditMember" xlink:to="pbhc_LinesOfCreditMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_LinesOfCreditMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Lines of Credit [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_LinesOfCreditMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Lines Of Credit [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_OtherCommercialAndIndustrialMember" xlink:type="locator" xlink:label="pbhc_OtherCommercialAndIndustrialMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_OtherCommercialAndIndustrialMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Class of financing receivables related to other commercial and industrial receivables.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_OtherCommercialAndIndustrialMember" xlink:to="pbhc_OtherCommercialAndIndustrialMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_OtherCommercialAndIndustrialMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Other Commercial and Industrial [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_OtherCommercialAndIndustrialMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Other Commercial And Industrial [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_TaxExemptLoansMember" xlink:type="locator" xlink:label="pbhc_TaxExemptLoansMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_TaxExemptLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Class of financing receivables related to tax exempt loans.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_TaxExemptLoansMember" xlink:to="pbhc_TaxExemptLoansMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_TaxExemptLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Tax Exempt Loans [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_TaxExemptLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Tax Exempt Loans [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_PaycheckProtectionProgramLoansMember" xlink:type="locator" xlink:label="pbhc_PaycheckProtectionProgramLoansMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_PaycheckProtectionProgramLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Paycheck Protection Program loans Member.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_PaycheckProtectionProgramLoansMember" xlink:to="pbhc_PaycheckProtectionProgramLoansMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_PaycheckProtectionProgramLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Paycheck Protection Program loans [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_PaycheckProtectionProgramLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Paycheck Protection Program Loans [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_HomeEquityAndJuniorLiensMember" xlink:type="locator" xlink:label="pbhc_HomeEquityAndJuniorLiensMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_HomeEquityAndJuniorLiensMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Class of financing receivables related to home equity and junior liens financing receivables.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_HomeEquityAndJuniorLiensMember" xlink:to="pbhc_HomeEquityAndJuniorLiensMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_HomeEquityAndJuniorLiensMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Home Equity and Junior Liens [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_HomeEquityAndJuniorLiensMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Home Equity And Junior Liens [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_OtherConsumerFinancingReceivableMember" xlink:type="locator" xlink:label="pbhc_OtherConsumerFinancingReceivableMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_OtherConsumerFinancingReceivableMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Class of financing receivables related to other consumer financing receivables.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_OtherConsumerFinancingReceivableMember" xlink:to="pbhc_OtherConsumerFinancingReceivableMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_OtherConsumerFinancingReceivableMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Other Consumer [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_OtherConsumerFinancingReceivableMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Other Consumer Financing Receivable [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AccountsNotesAndLoansReceivableLineItems" xlink:type="locator" xlink:label="us-gaap_AccountsNotesAndLoansReceivableLineItems" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AccountsNotesAndLoansReceivableLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Accounts Notes And Loans Receivable [Line Items]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AccountsNotesAndLoansReceivableLineItems" xlink:to="us-gaap_AccountsNotesAndLoansReceivableLineItems_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AccountsNotesAndLoansReceivableLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Accounts Notes And Loans Receivable [Line Items]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_NotesReceivableNetAbstract" xlink:type="locator" xlink:label="us-gaap_NotesReceivableNetAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_NotesReceivableNetAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Notes, Loans and Financing Receivable, Net [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_NotesReceivableNetAbstract" xlink:to="us-gaap_NotesReceivableNetAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_NotesReceivableNetAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Notes Receivable Net [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LoansAndLeasesReceivableGrossCarryingAmount" xlink:type="locator" xlink:label="us-gaap_LoansAndLeasesReceivableGrossCarryingAmount" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LoansAndLeasesReceivableGrossCarryingAmount_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Total loans</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LoansAndLeasesReceivableGrossCarryingAmount" xlink:to="us-gaap_LoansAndLeasesReceivableGrossCarryingAmount_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LoansAndLeasesReceivableGrossCarryingAmount_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Loans And Leases Receivable Gross Carrying Amount</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LoansAndLeasesReceivableDeferredIncome" xlink:type="locator" xlink:label="us-gaap_LoansAndLeasesReceivableDeferredIncome" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LoansAndLeasesReceivableDeferredIncome_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Net deferred loan (fees) costs</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LoansAndLeasesReceivableDeferredIncome" xlink:to="us-gaap_LoansAndLeasesReceivableDeferredIncome_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LoansAndLeasesReceivableDeferredIncome_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Loans And Leases Receivable Deferred Income</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_LoansAndLeasesReceivableAllowance_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Less allowance for loan losses</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_LoansAndLeasesReceivableAmortizedCost" xlink:type="locator" xlink:label="pbhc_LoansAndLeasesReceivableAmortizedCost" />
    <link:label xml:lang="en-US" xlink:label="pbhc_LoansAndLeasesReceivableAmortizedCost_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Loans and leases receivable amortized cost.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_LoansAndLeasesReceivableAmortizedCost" xlink:to="pbhc_LoansAndLeasesReceivableAmortizedCost_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_LoansAndLeasesReceivableAmortizedCost_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Loans and leases receivable amortized cost</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_LoansAndLeasesReceivableAmortizedCost_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Loans And Leases Receivable Amortized Cost</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_DebtInstrumentPrincipalOutstanding" xlink:type="locator" xlink:label="pbhc_DebtInstrumentPrincipalOutstanding" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DebtInstrumentPrincipalOutstanding_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Debt instrument principal outstanding.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_DebtInstrumentPrincipalOutstanding" xlink:to="pbhc_DebtInstrumentPrincipalOutstanding_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ParticipatingInterestOfAcquiredLoanPercentage" xlink:type="locator" xlink:label="pbhc_ParticipatingInterestOfAcquiredLoanPercentage" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ParticipatingInterestOfAcquiredLoanPercentage_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Participating interest of acquired loan percentage.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ParticipatingInterestOfAcquiredLoanPercentage" xlink:to="pbhc_ParticipatingInterestOfAcquiredLoanPercentage_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_NumberOfLoansOutstanding" xlink:type="locator" xlink:label="pbhc_NumberOfLoansOutstanding" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfLoansOutstanding_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Number of loans outstanding.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_NumberOfLoansOutstanding" xlink:to="pbhc_NumberOfLoansOutstanding_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_MaturityPeriodOfAcquiredLoan" xlink:type="locator" xlink:label="pbhc_MaturityPeriodOfAcquiredLoan" />
    <link:label xml:lang="en-US" xlink:label="pbhc_MaturityPeriodOfAcquiredLoan_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Refers to maturity period of loan receivable acquired, in PnYnMnDTnHnMnS&apos; format, for example, &apos;P1Y5M13D&apos; represents the reported fact of one year, five months, and thirteen days.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_MaturityPeriodOfAcquiredLoan" xlink:to="pbhc_MaturityPeriodOfAcquiredLoan_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ResidentialMortgageMember" xlink:type="locator" xlink:label="us-gaap_ResidentialMortgageMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ResidentialMortgageMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Residential Mortgage</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ResidentialMortgageMember" xlink:to="us-gaap_ResidentialMortgageMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ResidentialMortgageMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Residential Mortgage [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_CommercialLoanMember" xlink:type="locator" xlink:label="us-gaap_CommercialLoanMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CommercialLoanMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Commercial Loan</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_CommercialLoanMember" xlink:to="us-gaap_CommercialLoanMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CommercialLoanMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Commercial Loan [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_HomeEquityMember" xlink:type="locator" xlink:label="us-gaap_HomeEquityMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HomeEquityMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Home Equity Line of Credit [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_HomeEquityMember" xlink:to="us-gaap_HomeEquityMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HomeEquityMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Home Equity [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AutomobileLoanMember" xlink:type="locator" xlink:label="us-gaap_AutomobileLoanMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AutomobileLoanMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Automobile Loans [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AutomobileLoanMember" xlink:to="us-gaap_AutomobileLoanMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AutomobileLoanMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Automobile Loan [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_PurchasedUnsecuredConsumerLoanPool1Member" xlink:type="locator" xlink:label="pbhc_PurchasedUnsecuredConsumerLoanPool1Member" />
    <link:label xml:lang="en-US" xlink:label="pbhc_PurchasedUnsecuredConsumerLoanPool1Member_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Purchased unsecured consumer loan pool 1 Member.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_PurchasedUnsecuredConsumerLoanPool1Member" xlink:to="pbhc_PurchasedUnsecuredConsumerLoanPool1Member_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_PurchasedUnsecuredConsumerLoanPool1Member_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Purchased Unsecured Consumer Loan Pool1</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_PurchasedUnsecuredConsumerLoanPool1Member_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Purchased Unsecured Consumer Loan Pool1 [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_PurchasedUnsecuredConsumerLoanPool2Member" xlink:type="locator" xlink:label="pbhc_PurchasedUnsecuredConsumerLoanPool2Member" />
    <link:label xml:lang="en-US" xlink:label="pbhc_PurchasedUnsecuredConsumerLoanPool2Member_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Purchased unsecured consumer loan pool 2 Member.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_PurchasedUnsecuredConsumerLoanPool2Member" xlink:to="pbhc_PurchasedUnsecuredConsumerLoanPool2Member_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_PurchasedUnsecuredConsumerLoanPool2Member_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Purchased Unsecured Consumer Loan Pool2</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_PurchasedUnsecuredConsumerLoanPool2Member_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Purchased Unsecured Consumer Loan Pool2 [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_PurchasedUnsecuredConsumerLoanPool3Member" xlink:type="locator" xlink:label="pbhc_PurchasedUnsecuredConsumerLoanPool3Member" />
    <link:label xml:lang="en-US" xlink:label="pbhc_PurchasedUnsecuredConsumerLoanPool3Member_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Purchased unsecured consumer loan pool 3 Member.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_PurchasedUnsecuredConsumerLoanPool3Member" xlink:to="pbhc_PurchasedUnsecuredConsumerLoanPool3Member_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_PurchasedUnsecuredConsumerLoanPool3Member_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Purchased Unsecured Consumer Loan Pool3</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_PurchasedUnsecuredConsumerLoanPool3Member_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Purchased Unsecured Consumer Loan Pool3 [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_PurchasedUnsecuredConsumerLoanPool4Member" xlink:type="locator" xlink:label="pbhc_PurchasedUnsecuredConsumerLoanPool4Member" />
    <link:label xml:lang="en-US" xlink:label="pbhc_PurchasedUnsecuredConsumerLoanPool4Member_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Purchased unsecured consumer loan pool 4.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_PurchasedUnsecuredConsumerLoanPool4Member" xlink:to="pbhc_PurchasedUnsecuredConsumerLoanPool4Member_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_PurchasedUnsecuredConsumerLoanPool4Member_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Purchased Unsecured Consumer Loan Pool4</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_PurchasedUnsecuredConsumerLoanPool4Member_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Purchased Unsecured Consumer Loan Pool4 [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DebtInstrumentPrincipalOutstanding_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Original Balance</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DebtInstrumentPrincipalOutstanding_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Debt Instrument Principal Outstanding</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_CertainLoansAcquiredInTransferNotAccountedForAsDebtSecuritiesCarryingAmountNet" xlink:type="locator" xlink:label="us-gaap_CertainLoansAcquiredInTransferNotAccountedForAsDebtSecuritiesCarryingAmountNet" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CertainLoansAcquiredInTransferNotAccountedForAsDebtSecuritiesCarryingAmountNet_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Current Balance</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_CertainLoansAcquiredInTransferNotAccountedForAsDebtSecuritiesCarryingAmountNet" xlink:to="us-gaap_CertainLoansAcquiredInTransferNotAccountedForAsDebtSecuritiesCarryingAmountNet_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CertainLoansAcquiredInTransferNotAccountedForAsDebtSecuritiesCarryingAmountNet_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Certain Loans Acquired In Transfer Not Accounted For As Debt Securities Carrying Amount Net</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DebtInstrumentUnamortizedPremium" xlink:type="locator" xlink:label="us-gaap_DebtInstrumentUnamortizedPremium" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtInstrumentUnamortizedPremium_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Unamortized Premium</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DebtInstrumentUnamortizedPremium" xlink:to="us-gaap_DebtInstrumentUnamortizedPremium_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtInstrumentUnamortizedPremium_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Debt Instrument Unamortized Premium</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ParticipatingInterestOfAcquiredLoanPercentage_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Percent Owned</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ParticipatingInterestOfAcquiredLoanPercentage_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Participating Interest Of Acquired Loan Percentage</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfLoansOutstanding_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Number of Loans</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfLoansOutstanding_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Number Of Loans Outstanding</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MaturityPeriodOfAcquiredLoan_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Maturity range</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MaturityPeriodOfAcquiredLoan_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Maturity Period Of Acquired Loan</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AllowanceForLoanAndLeaseLossesWriteoffsNet" xlink:type="locator" xlink:label="us-gaap_AllowanceForLoanAndLeaseLossesWriteoffsNet" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AllowanceForLoanAndLeaseLossesWriteoffsNet_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Cumulative net charge-offs</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AllowanceForLoanAndLeaseLossesWriteoffsNet" xlink:to="us-gaap_AllowanceForLoanAndLeaseLossesWriteoffsNet_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AllowanceForLoanAndLeaseLossesWriteoffsNet_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Allowance For Loan And Lease Losses Writeoffs Net</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_TroubledDebtRestructuringDebtorSubsequentPeriodsAxis" xlink:type="locator" xlink:label="us-gaap_TroubledDebtRestructuringDebtorSubsequentPeriodsAxis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TroubledDebtRestructuringDebtorSubsequentPeriodsAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Troubled Debt Restructuring, Debtor, Subsequent Periods</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_TroubledDebtRestructuringDebtorSubsequentPeriodsAxis" xlink:to="us-gaap_TroubledDebtRestructuringDebtorSubsequentPeriodsAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TroubledDebtRestructuringDebtorSubsequentPeriodsAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Troubled Debt Restructuring Debtor Subsequent Periods [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_TroubledDebtRestructuringDebtorSubsequentPeriodsContingentPaymentsNameDomain" xlink:type="locator" xlink:label="us-gaap_TroubledDebtRestructuringDebtorSubsequentPeriodsContingentPaymentsNameDomain" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TroubledDebtRestructuringDebtorSubsequentPeriodsContingentPaymentsNameDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Troubled Debt Restructuring, Debtor, Subsequent Periods, Contingent Payments, Name</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_TroubledDebtRestructuringDebtorSubsequentPeriodsContingentPaymentsNameDomain" xlink:to="us-gaap_TroubledDebtRestructuringDebtorSubsequentPeriodsContingentPaymentsNameDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TroubledDebtRestructuringDebtorSubsequentPeriodsContingentPaymentsNameDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Troubled Debt Restructuring Debtor Subsequent Periods Contingent Payments Name [Domain]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_TroubledDebtRestructuringDuringPriorTwelveMonthsMember" xlink:type="locator" xlink:label="pbhc_TroubledDebtRestructuringDuringPriorTwelveMonthsMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_TroubledDebtRestructuringDuringPriorTwelveMonthsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Troubled debt restructuring during prior twelve months.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_TroubledDebtRestructuringDuringPriorTwelveMonthsMember" xlink:to="pbhc_TroubledDebtRestructuringDuringPriorTwelveMonthsMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_TroubledDebtRestructuringDuringPriorTwelveMonthsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Troubled Debt Restructuring During Prior Twelve Months [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_TroubledDebtRestructuringDuringPriorTwelveMonthsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Troubled Debt Restructuring During Prior Twelve Months [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LoansPledgedAsCollateral" xlink:type="locator" xlink:label="us-gaap_LoansPledgedAsCollateral" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LoansPledgedAsCollateral_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Residential mortgage loans pledged to FHLBNY as blanket collateral</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LoansPledgedAsCollateral" xlink:to="us-gaap_LoansPledgedAsCollateral_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LoansPledgedAsCollateral_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Loans Pledged As Collateral</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_PercentageOfTotalLoanPortfolio" xlink:type="locator" xlink:label="pbhc_PercentageOfTotalLoanPortfolio" />
    <link:label xml:lang="en-US" xlink:label="pbhc_PercentageOfTotalLoanPortfolio_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The percentage of the total loan portfolio.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_PercentageOfTotalLoanPortfolio" xlink:to="pbhc_PercentageOfTotalLoanPortfolio_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_PercentageOfTotalLoanPortfolio_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Percentage of total loan portfolio</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_PercentageOfTotalLoanPortfolio_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Percentage Of Total Loan Portfolio</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_NumberOfDaysPastDueBeforeLoanIsClassifiedAsSpecialMention" xlink:type="locator" xlink:label="pbhc_NumberOfDaysPastDueBeforeLoanIsClassifiedAsSpecialMention" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfDaysPastDueBeforeLoanIsClassifiedAsSpecialMention_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The number of days past due before a loan is classified as special mention.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_NumberOfDaysPastDueBeforeLoanIsClassifiedAsSpecialMention" xlink:to="pbhc_NumberOfDaysPastDueBeforeLoanIsClassifiedAsSpecialMention_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_NumberOfConsecutiveMonthsCurrentBeforeLoanIsUpgraded" xlink:type="locator" xlink:label="pbhc_NumberOfConsecutiveMonthsCurrentBeforeLoanIsUpgraded" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfConsecutiveMonthsCurrentBeforeLoanIsUpgraded_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The number of consecutive months that a loan is classified as current before the loan status is upgraded.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_NumberOfConsecutiveMonthsCurrentBeforeLoanIsUpgraded" xlink:to="pbhc_NumberOfConsecutiveMonthsCurrentBeforeLoanIsUpgraded_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ScheduleOfFinancingReceivableRecordedInvestmentCreditQualityIndicatorTable" xlink:type="locator" xlink:label="us-gaap_ScheduleOfFinancingReceivableRecordedInvestmentCreditQualityIndicatorTable" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfFinancingReceivableRecordedInvestmentCreditQualityIndicatorTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Schedule Of Financing Receivable Recorded Investment Credit Quality Indicator [Table]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ScheduleOfFinancingReceivableRecordedInvestmentCreditQualityIndicatorTable" xlink:to="us-gaap_ScheduleOfFinancingReceivableRecordedInvestmentCreditQualityIndicatorTable_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfFinancingReceivableRecordedInvestmentCreditQualityIndicatorTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Financing Receivable Recorded Investment Credit Quality Indicator [Table]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ResidentialMortgageAndConsumerLoansMember" xlink:type="locator" xlink:label="pbhc_ResidentialMortgageAndConsumerLoansMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ResidentialMortgageAndConsumerLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Class of financing receivables related to residential mortgage and consumer loans combined.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ResidentialMortgageAndConsumerLoansMember" xlink:to="pbhc_ResidentialMortgageAndConsumerLoansMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ResidentialMortgageAndConsumerLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Residential Mortgage and Consumer Loans [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ResidentialMortgageAndConsumerLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Residential Mortgage And Consumer Loans [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_SpecialMentionMember" xlink:type="locator" xlink:label="us-gaap_SpecialMentionMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SpecialMentionMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Special Mention [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_SpecialMentionMember" xlink:to="us-gaap_SpecialMentionMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SpecialMentionMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Special Mention [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_SubstandardMember" xlink:type="locator" xlink:label="us-gaap_SubstandardMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SubstandardMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Substandard [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_SubstandardMember" xlink:to="us-gaap_SubstandardMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SubstandardMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Substandard [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FinancingReceivableRecordedInvestmentLineItems" xlink:type="locator" xlink:label="us-gaap_FinancingReceivableRecordedInvestmentLineItems" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableRecordedInvestmentLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Financing Receivable Recorded Investment [Line Items]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FinancingReceivableRecordedInvestmentLineItems" xlink:to="us-gaap_FinancingReceivableRecordedInvestmentLineItems_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableRecordedInvestmentLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Financing Receivable Recorded Investment [Line Items]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfDaysPastDueBeforeLoanIsClassifiedAsSpecialMention_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Number of days past due before a loan is classified as Special Mention</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfDaysPastDueBeforeLoanIsClassifiedAsSpecialMention_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Number Of Days Past Due Before Loan Is Classified As Special Mention</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfConsecutiveMonthsCurrentBeforeLoanIsUpgraded_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Number of consecutive months current before loan is upgraded</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfConsecutiveMonthsCurrentBeforeLoanIsUpgraded_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Number Of Consecutive Months Current Before Loan Is Upgraded</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_NumberOfDaysPastDueBeforeLoanIsClassifiedAsSubstandard" xlink:type="locator" xlink:label="pbhc_NumberOfDaysPastDueBeforeLoanIsClassifiedAsSubstandard" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfDaysPastDueBeforeLoanIsClassifiedAsSubstandard_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The number of days past due before a loan is classified as substandard.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_NumberOfDaysPastDueBeforeLoanIsClassifiedAsSubstandard" xlink:to="pbhc_NumberOfDaysPastDueBeforeLoanIsClassifiedAsSubstandard_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfDaysPastDueBeforeLoanIsClassifiedAsSubstandard_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Number of days past due before a loan is classified as Substandard</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfDaysPastDueBeforeLoanIsClassifiedAsSubstandard_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Number Of Days Past Due Before Loan Is Classified As Substandard</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_PassMember" xlink:type="locator" xlink:label="us-gaap_PassMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PassMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Pass [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_PassMember" xlink:to="us-gaap_PassMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PassMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Pass [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DoubtfulMember" xlink:type="locator" xlink:label="us-gaap_DoubtfulMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DoubtfulMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Doubtful [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DoubtfulMember" xlink:to="us-gaap_DoubtfulMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DoubtfulMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Doubtful [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_LoanHeldForSaleMember" xlink:type="locator" xlink:label="pbhc_LoanHeldForSaleMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_LoanHeldForSaleMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Loan held for sale.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_LoanHeldForSaleMember" xlink:to="pbhc_LoanHeldForSaleMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_LoanHeldForSaleMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Loan Held For Sale</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_LoanHeldForSaleMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Loan Held For Sale [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ScheduleOfFinancingReceivablesPastDueTable" xlink:type="locator" xlink:label="us-gaap_ScheduleOfFinancingReceivablesPastDueTable" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfFinancingReceivablesPastDueTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Schedule Of Financing Receivables Past Due [Table]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ScheduleOfFinancingReceivablesPastDueTable" xlink:to="us-gaap_ScheduleOfFinancingReceivablesPastDueTable_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfFinancingReceivablesPastDueTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Financing Receivables Past Due [Table]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FinancingReceivablesPeriodPastDueAxis" xlink:type="locator" xlink:label="us-gaap_FinancingReceivablesPeriodPastDueAxis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivablesPeriodPastDueAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Financial Asset, Period Past Due</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FinancingReceivablesPeriodPastDueAxis" xlink:to="us-gaap_FinancingReceivablesPeriodPastDueAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivablesPeriodPastDueAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Financing Receivables Period Past Due [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FinancingReceivablesPeriodPastDueDomain" xlink:type="locator" xlink:label="us-gaap_FinancingReceivablesPeriodPastDueDomain" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivablesPeriodPastDueDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Financial Asset, Period Past Due</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FinancingReceivablesPeriodPastDueDomain" xlink:to="us-gaap_FinancingReceivablesPeriodPastDueDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivablesPeriodPastDueDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Financing Receivables Period Past Due [Domain]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FinancingReceivables30To59DaysPastDueMember" xlink:type="locator" xlink:label="us-gaap_FinancingReceivables30To59DaysPastDueMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivables30To59DaysPastDueMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">30-59 Days Past Due and Accruing [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FinancingReceivables30To59DaysPastDueMember" xlink:to="us-gaap_FinancingReceivables30To59DaysPastDueMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivables30To59DaysPastDueMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Financing Receivables30 To59 Days Past Due [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FinancingReceivables60To89DaysPastDueMember" xlink:type="locator" xlink:label="us-gaap_FinancingReceivables60To89DaysPastDueMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivables60To89DaysPastDueMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">60-89 Days Past Due and Accruing[Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FinancingReceivables60To89DaysPastDueMember" xlink:to="us-gaap_FinancingReceivables60To89DaysPastDueMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivables60To89DaysPastDueMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Financing Receivables60 To89 Days Past Due [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FinancingReceivablesEqualToGreaterThan90DaysPastDueMember" xlink:type="locator" xlink:label="us-gaap_FinancingReceivablesEqualToGreaterThan90DaysPastDueMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivablesEqualToGreaterThan90DaysPastDueMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">90 Days and Over [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FinancingReceivablesEqualToGreaterThan90DaysPastDueMember" xlink:to="us-gaap_FinancingReceivablesEqualToGreaterThan90DaysPastDueMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivablesEqualToGreaterThan90DaysPastDueMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Financing Receivables Equal To Greater Than90 Days Past Due [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FinancingReceivableRecordedInvestmentPastDueLineItems" xlink:type="locator" xlink:label="us-gaap_FinancingReceivableRecordedInvestmentPastDueLineItems" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableRecordedInvestmentPastDueLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Recorded Investment, Past Due [Line Items]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FinancingReceivableRecordedInvestmentPastDueLineItems" xlink:to="us-gaap_FinancingReceivableRecordedInvestmentPastDueLineItems_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableRecordedInvestmentPastDueLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Financing Receivable Recorded Investment Past Due [Line Items]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FinancingReceivableRecordedInvestmentPastDue" xlink:type="locator" xlink:label="us-gaap_FinancingReceivableRecordedInvestmentPastDue" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableRecordedInvestmentPastDue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Total Past Due</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FinancingReceivableRecordedInvestmentPastDue" xlink:to="us-gaap_FinancingReceivableRecordedInvestmentPastDue_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableRecordedInvestmentPastDue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Financing Receivable Recorded Investment Past Due</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FinancingReceivableRecordedInvestmentCurrent" xlink:type="locator" xlink:label="us-gaap_FinancingReceivableRecordedInvestmentCurrent" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableRecordedInvestmentCurrent_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Current</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FinancingReceivableRecordedInvestmentCurrent" xlink:to="us-gaap_FinancingReceivableRecordedInvestmentCurrent_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableRecordedInvestmentCurrent_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Financing Receivable Recorded Investment Current</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_LoansAndLeasesReceivableGrossCarryingAmount_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Total Loans Receivable</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_FamilyFirstLienResidentialMortgages1To4Member" xlink:type="locator" xlink:label="pbhc_FamilyFirstLienResidentialMortgages1To4Member" />
    <link:label xml:lang="en-US" xlink:label="pbhc_FamilyFirstLienResidentialMortgages1To4Member_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Class of financing receivables related to 1-4 family first-lien residential mortgages.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_FamilyFirstLienResidentialMortgages1To4Member" xlink:to="pbhc_FamilyFirstLienResidentialMortgages1To4Member_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_FamilyFirstLienResidentialMortgages1To4Member_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">1-4 Family First Lien Residential Mortgages [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_FamilyFirstLienResidentialMortgages1To4Member_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Family First Lien Residential Mortgages1 To4 [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FinancingReceivableRecordedInvestmentAgingAbstract" xlink:type="locator" xlink:label="us-gaap_FinancingReceivableRecordedInvestmentAgingAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableRecordedInvestmentAgingAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Nonaccrual loans, Segregated by class of loans [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FinancingReceivableRecordedInvestmentAgingAbstract" xlink:to="us-gaap_FinancingReceivableRecordedInvestmentAgingAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableRecordedInvestmentAgingAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Financing Receivable Recorded Investment Aging [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FinancingReceivableRecordedInvestmentNonaccrualStatus" xlink:type="locator" xlink:label="us-gaap_FinancingReceivableRecordedInvestmentNonaccrualStatus" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableRecordedInvestmentNonaccrualStatus_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Nonaccrual status loans</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FinancingReceivableRecordedInvestmentNonaccrualStatus" xlink:to="us-gaap_FinancingReceivableRecordedInvestmentNonaccrualStatus_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableRecordedInvestmentNonaccrualStatus_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Financing Receivable Recorded Investment Nonaccrual Status</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FinancingReceivableRecordedInvestment90DaysPastDueAndStillAccruing" xlink:type="locator" xlink:label="us-gaap_FinancingReceivableRecordedInvestment90DaysPastDueAndStillAccruing" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableRecordedInvestment90DaysPastDueAndStillAccruing_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Ninety days past due and still accruing interest</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FinancingReceivableRecordedInvestment90DaysPastDueAndStillAccruing" xlink:to="us-gaap_FinancingReceivableRecordedInvestment90DaysPastDueAndStillAccruing_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableRecordedInvestment90DaysPastDueAndStillAccruing_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Financing Receivable Recorded Investment90 Days Past Due And Still Accruing</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_CommercialRealEstateMember" xlink:type="locator" xlink:label="us-gaap_CommercialRealEstateMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CommercialRealEstateMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Commercial Real Estate Loans [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_CommercialRealEstateMember" xlink:to="us-gaap_CommercialRealEstateMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CommercialRealEstateMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Commercial Real Estate [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_CommercialAndIndustrialMember" xlink:type="locator" xlink:label="pbhc_CommercialAndIndustrialMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_CommercialAndIndustrialMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Commercial and industrial.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_CommercialAndIndustrialMember" xlink:to="pbhc_CommercialAndIndustrialMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_CommercialAndIndustrialMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Commercial and Industrial Loans [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_CommercialAndIndustrialMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Commercial And Industrial [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ResidentialRealEstateMember" xlink:type="locator" xlink:label="us-gaap_ResidentialRealEstateMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ResidentialRealEstateMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Residential Real Estate Loans [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ResidentialRealEstateMember" xlink:to="us-gaap_ResidentialRealEstateMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ResidentialRealEstateMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Residential Real Estate [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_TroubledDebtRestructuringDebtorCurrentPeriodAbstract" xlink:type="locator" xlink:label="us-gaap_TroubledDebtRestructuringDebtorCurrentPeriodAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TroubledDebtRestructuringDebtorCurrentPeriodAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Troubled Debt Restructuring, Modified [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_TroubledDebtRestructuringDebtorCurrentPeriodAbstract" xlink:to="us-gaap_TroubledDebtRestructuringDebtorCurrentPeriodAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TroubledDebtRestructuringDebtorCurrentPeriodAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Troubled Debt Restructuring Debtor Current Period [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FinancingReceivableModificationsNumberOfContracts2" xlink:type="locator" xlink:label="us-gaap_FinancingReceivableModificationsNumberOfContracts2" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableModificationsNumberOfContracts2_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Number of loans</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FinancingReceivableModificationsNumberOfContracts2" xlink:to="us-gaap_FinancingReceivableModificationsNumberOfContracts2_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableModificationsNumberOfContracts2_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Financing Receivable Modifications Number Of Contracts2</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FinancingReceivableModificationsPreModificationRecordedInvestment2" xlink:type="locator" xlink:label="us-gaap_FinancingReceivableModificationsPreModificationRecordedInvestment2" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableModificationsPreModificationRecordedInvestment2_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Pre-modification outstanding recorded investment</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FinancingReceivableModificationsPreModificationRecordedInvestment2" xlink:to="us-gaap_FinancingReceivableModificationsPreModificationRecordedInvestment2_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableModificationsPreModificationRecordedInvestment2_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Financing Receivable Modifications Pre Modification Recorded Investment2</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FinancingReceivableModificationsPostModificationRecordedInvestment2" xlink:type="locator" xlink:label="us-gaap_FinancingReceivableModificationsPostModificationRecordedInvestment2" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableModificationsPostModificationRecordedInvestment2_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Post-modification outstanding recorded investment</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FinancingReceivableModificationsPostModificationRecordedInvestment2" xlink:to="us-gaap_FinancingReceivableModificationsPostModificationRecordedInvestment2_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableModificationsPostModificationRecordedInvestment2_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Financing Receivable Modifications Post Modification Recorded Investment2</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ProvisionForLoanLossesExpensed" xlink:type="locator" xlink:label="us-gaap_ProvisionForLoanLossesExpensed" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ProvisionForLoanLossesExpensed_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Additional provision for loan losses</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ProvisionForLoanLossesExpensed" xlink:to="us-gaap_ProvisionForLoanLossesExpensed_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ProvisionForLoanLossesExpensed_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Provision For Loan Losses Expensed</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FinancingReceivableModificationsSubsequentDefaultNumberOfContracts1" xlink:type="locator" xlink:label="us-gaap_FinancingReceivableModificationsSubsequentDefaultNumberOfContracts1" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableModificationsSubsequentDefaultNumberOfContracts1_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Number of loans subsequently defaulted</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FinancingReceivableModificationsSubsequentDefaultNumberOfContracts1" xlink:to="us-gaap_FinancingReceivableModificationsSubsequentDefaultNumberOfContracts1_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableModificationsSubsequentDefaultNumberOfContracts1_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Financing Receivable Modifications Subsequent Default Number Of Contracts1</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_WithNoRelatedAllowanceRecordedAbstract" xlink:type="locator" xlink:label="pbhc_WithNoRelatedAllowanceRecordedAbstract" />
    <link:label xml:lang="en-US" xlink:label="pbhc_WithNoRelatedAllowanceRecordedAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">With no related allowance recorded.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_WithNoRelatedAllowanceRecordedAbstract" xlink:to="pbhc_WithNoRelatedAllowanceRecordedAbstract_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ScheduleOfImpairedFinancingReceivableTable" xlink:type="locator" xlink:label="us-gaap_ScheduleOfImpairedFinancingReceivableTable" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfImpairedFinancingReceivableTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Schedule Of Impaired Financing Receivable [Table]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ScheduleOfImpairedFinancingReceivableTable" xlink:to="us-gaap_ScheduleOfImpairedFinancingReceivableTable_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfImpairedFinancingReceivableTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Impaired Financing Receivable [Table]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FinancingReceivableImpairedLineItems" xlink:type="locator" xlink:label="us-gaap_FinancingReceivableImpairedLineItems" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableImpairedLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Financing Receivable Impaired [Line Items]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FinancingReceivableImpairedLineItems" xlink:to="us-gaap_FinancingReceivableImpairedLineItems_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableImpairedLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Financing Receivable Impaired [Line Items]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_WithNoRelatedAllowanceRecordedAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">With no related allowance recorded [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_WithNoRelatedAllowanceRecordedAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">With No Related Allowance Recorded [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ImpairedFinancingReceivableWithNoRelatedAllowanceRecordedInvestment" xlink:type="locator" xlink:label="us-gaap_ImpairedFinancingReceivableWithNoRelatedAllowanceRecordedInvestment" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ImpairedFinancingReceivableWithNoRelatedAllowanceRecordedInvestment_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Recorded Investment</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ImpairedFinancingReceivableWithNoRelatedAllowanceRecordedInvestment" xlink:to="us-gaap_ImpairedFinancingReceivableWithNoRelatedAllowanceRecordedInvestment_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ImpairedFinancingReceivableWithNoRelatedAllowanceRecordedInvestment_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Impaired Financing Receivable With No Related Allowance Recorded Investment</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ImpairedFinancingReceivableWithNoRelatedAllowanceUnpaidPrincipalBalance" xlink:type="locator" xlink:label="us-gaap_ImpairedFinancingReceivableWithNoRelatedAllowanceUnpaidPrincipalBalance" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ImpairedFinancingReceivableWithNoRelatedAllowanceUnpaidPrincipalBalance_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Unpaid Principal Balance</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ImpairedFinancingReceivableWithNoRelatedAllowanceUnpaidPrincipalBalance" xlink:to="us-gaap_ImpairedFinancingReceivableWithNoRelatedAllowanceUnpaidPrincipalBalance_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ImpairedFinancingReceivableWithNoRelatedAllowanceUnpaidPrincipalBalance_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Impaired Financing Receivable With No Related Allowance Unpaid Principal Balance</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_WithAllowanceRecordedAbstract" xlink:type="locator" xlink:label="pbhc_WithAllowanceRecordedAbstract" />
    <link:label xml:lang="en-US" xlink:label="pbhc_WithAllowanceRecordedAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">With allowance recorded.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_WithAllowanceRecordedAbstract" xlink:to="pbhc_WithAllowanceRecordedAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_WithAllowanceRecordedAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">With an allowance recorded [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_WithAllowanceRecordedAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">With Allowance Recorded [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ImpairedFinancingReceivableWithRelatedAllowanceRecordedInvestment" xlink:type="locator" xlink:label="us-gaap_ImpairedFinancingReceivableWithRelatedAllowanceRecordedInvestment" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ImpairedFinancingReceivableWithRelatedAllowanceRecordedInvestment_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Recorded Investment</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ImpairedFinancingReceivableWithRelatedAllowanceRecordedInvestment" xlink:to="us-gaap_ImpairedFinancingReceivableWithRelatedAllowanceRecordedInvestment_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ImpairedFinancingReceivableWithRelatedAllowanceRecordedInvestment_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Impaired Financing Receivable With Related Allowance Recorded Investment</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ImpairedFinancingReceivableWithRelatedAllowanceUnpaidPrincipalBalance" xlink:type="locator" xlink:label="us-gaap_ImpairedFinancingReceivableWithRelatedAllowanceUnpaidPrincipalBalance" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ImpairedFinancingReceivableWithRelatedAllowanceUnpaidPrincipalBalance_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Unpaid Principal Balance</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ImpairedFinancingReceivableWithRelatedAllowanceUnpaidPrincipalBalance" xlink:to="us-gaap_ImpairedFinancingReceivableWithRelatedAllowanceUnpaidPrincipalBalance_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ImpairedFinancingReceivableWithRelatedAllowanceUnpaidPrincipalBalance_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Impaired Financing Receivable With Related Allowance Unpaid Principal Balance</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ImpairedFinancingReceivableRelatedAllowance" xlink:type="locator" xlink:label="us-gaap_ImpairedFinancingReceivableRelatedAllowance" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ImpairedFinancingReceivableRelatedAllowance_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Related Allowance</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ImpairedFinancingReceivableRelatedAllowance" xlink:to="us-gaap_ImpairedFinancingReceivableRelatedAllowance_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ImpairedFinancingReceivableRelatedAllowance_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Impaired Financing Receivable Related Allowance</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_TotalAbstract" xlink:type="locator" xlink:label="pbhc_TotalAbstract" />
    <link:label xml:lang="en-US" xlink:label="pbhc_TotalAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Total.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_TotalAbstract" xlink:to="pbhc_TotalAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_TotalAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Total [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_TotalAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Total [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ImpairedFinancingReceivableRecordedInvestment" xlink:type="locator" xlink:label="us-gaap_ImpairedFinancingReceivableRecordedInvestment" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ImpairedFinancingReceivableRecordedInvestment_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Recorded Investment</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ImpairedFinancingReceivableRecordedInvestment" xlink:to="us-gaap_ImpairedFinancingReceivableRecordedInvestment_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ImpairedFinancingReceivableRecordedInvestment_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Impaired Financing Receivable Recorded Investment</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ImpairedFinancingReceivableUnpaidPrincipalBalance" xlink:type="locator" xlink:label="us-gaap_ImpairedFinancingReceivableUnpaidPrincipalBalance" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ImpairedFinancingReceivableUnpaidPrincipalBalance_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Unpaid Principal Balance</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ImpairedFinancingReceivableUnpaidPrincipalBalance" xlink:to="us-gaap_ImpairedFinancingReceivableUnpaidPrincipalBalance_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ImpairedFinancingReceivableUnpaidPrincipalBalance_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Impaired Financing Receivable Unpaid Principal Balance</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_RealEstateLoanMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Commercial Real Estate [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_LinesOfCreditMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Commercial Lines of Credit [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ImpairedFinancingReceivableAverageRecordedInvestmentAbstract" xlink:type="locator" xlink:label="us-gaap_ImpairedFinancingReceivableAverageRecordedInvestmentAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ImpairedFinancingReceivableAverageRecordedInvestmentAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Average recorded investment [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ImpairedFinancingReceivableAverageRecordedInvestmentAbstract" xlink:to="us-gaap_ImpairedFinancingReceivableAverageRecordedInvestmentAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ImpairedFinancingReceivableAverageRecordedInvestmentAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Impaired Financing Receivable Average Recorded Investment [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ImpairedFinancingReceivableAverageRecordedInvestment" xlink:type="locator" xlink:label="us-gaap_ImpairedFinancingReceivableAverageRecordedInvestment" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ImpairedFinancingReceivableAverageRecordedInvestment_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Average recorded investment in impaired loans</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ImpairedFinancingReceivableAverageRecordedInvestment" xlink:to="us-gaap_ImpairedFinancingReceivableAverageRecordedInvestment_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ImpairedFinancingReceivableAverageRecordedInvestment_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Impaired Financing Receivable Average Recorded Investment</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ImpairedFinancingReceivableInterestIncomeCashBasisMethodAbstract" xlink:type="locator" xlink:label="us-gaap_ImpairedFinancingReceivableInterestIncomeCashBasisMethodAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ImpairedFinancingReceivableInterestIncomeCashBasisMethodAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Cash Basis Interest Recognized on Impaired Loans [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ImpairedFinancingReceivableInterestIncomeCashBasisMethodAbstract" xlink:to="us-gaap_ImpairedFinancingReceivableInterestIncomeCashBasisMethodAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ImpairedFinancingReceivableInterestIncomeCashBasisMethodAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Impaired Financing Receivable Interest Income Cash Basis Method [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ImpairedFinancingReceivableInterestIncomeCashBasisMethod" xlink:type="locator" xlink:label="us-gaap_ImpairedFinancingReceivableInterestIncomeCashBasisMethod" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ImpairedFinancingReceivableInterestIncomeCashBasisMethod_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Cash basis interest income recognized on impaired loans</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ImpairedFinancingReceivableInterestIncomeCashBasisMethod" xlink:to="us-gaap_ImpairedFinancingReceivableInterestIncomeCashBasisMethod_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ImpairedFinancingReceivableInterestIncomeCashBasisMethod_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Impaired Financing Receivable Interest Income Cash Basis Method</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ScheduleOfFinancingReceivableAllowanceForCreditLossesTable" xlink:type="locator" xlink:label="us-gaap_ScheduleOfFinancingReceivableAllowanceForCreditLossesTable" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfFinancingReceivableAllowanceForCreditLossesTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Schedule Of Financing Receivable Allowance For Credit Losses [Table]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ScheduleOfFinancingReceivableAllowanceForCreditLossesTable" xlink:to="us-gaap_ScheduleOfFinancingReceivableAllowanceForCreditLossesTable_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfFinancingReceivableAllowanceForCreditLossesTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Financing Receivable Allowance For Credit Losses [Table]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ResidentialMortgageConstructionMember" xlink:type="locator" xlink:label="pbhc_ResidentialMortgageConstructionMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ResidentialMortgageConstructionMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Residential mortgage construction.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ResidentialMortgageConstructionMember" xlink:to="pbhc_ResidentialMortgageConstructionMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ResidentialMortgageConstructionMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Residential Construction Mortgage [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ResidentialMortgageConstructionMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Residential Mortgage Construction [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_PaycheckProtectionProgramMember" xlink:type="locator" xlink:label="pbhc_PaycheckProtectionProgramMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_PaycheckProtectionProgramMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Paycheck Protection Program.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_PaycheckProtectionProgramMember" xlink:to="pbhc_PaycheckProtectionProgramMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_PaycheckProtectionProgramMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Paycheck Protection Program [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_PaycheckProtectionProgramMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Paycheck Protection Program [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_UnallocatedMember" xlink:type="locator" xlink:label="pbhc_UnallocatedMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_UnallocatedMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Class of financing receivables related to unallocated financing receivables.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_UnallocatedMember" xlink:to="pbhc_UnallocatedMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_UnallocatedMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Unallocated [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_UnallocatedMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Unallocated [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FinancingReceivableAllowanceForCreditLossesLineItems" xlink:type="locator" xlink:label="us-gaap_FinancingReceivableAllowanceForCreditLossesLineItems" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableAllowanceForCreditLossesLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Financing Receivable Allowance For Credit Losses [Line Items]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FinancingReceivableAllowanceForCreditLossesLineItems" xlink:to="us-gaap_FinancingReceivableAllowanceForCreditLossesLineItems_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableAllowanceForCreditLossesLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Financing Receivable Allowance For Credit Losses [Line Items]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FinancingReceivableAllowanceForCreditLossesRollForward" xlink:type="locator" xlink:label="us-gaap_FinancingReceivableAllowanceForCreditLossesRollForward" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableAllowanceForCreditLossesRollForward_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Allowance for loan losses:</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FinancingReceivableAllowanceForCreditLossesRollForward" xlink:to="us-gaap_FinancingReceivableAllowanceForCreditLossesRollForward_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableAllowanceForCreditLossesRollForward_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Financing Receivable Allowance For Credit Losses Roll Forward</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FinancingReceivableAllowanceForCreditLosses" xlink:type="locator" xlink:label="us-gaap_FinancingReceivableAllowanceForCreditLosses" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableAllowanceForCreditLosses_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/periodStartLabel">Beginning Balance</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FinancingReceivableAllowanceForCreditLosses" xlink:to="us-gaap_FinancingReceivableAllowanceForCreditLosses_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableAllowanceForCreditLosses_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Financing Receivable Allowance For Credit Losses</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FinancingReceivableAllowanceForCreditLossesWriteOffs" xlink:type="locator" xlink:label="us-gaap_FinancingReceivableAllowanceForCreditLossesWriteOffs" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableAllowanceForCreditLossesWriteOffs_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Charge-offs</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FinancingReceivableAllowanceForCreditLossesWriteOffs" xlink:to="us-gaap_FinancingReceivableAllowanceForCreditLossesWriteOffs_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableAllowanceForCreditLossesWriteOffs_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Financing Receivable Allowance For Credit Losses Write Offs</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FinancingReceivableAllowanceForCreditLossesRecovery" xlink:type="locator" xlink:label="us-gaap_FinancingReceivableAllowanceForCreditLossesRecovery" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableAllowanceForCreditLossesRecovery_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Recoveries</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FinancingReceivableAllowanceForCreditLossesRecovery" xlink:to="us-gaap_FinancingReceivableAllowanceForCreditLossesRecovery_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableAllowanceForCreditLossesRecovery_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Financing Receivable Allowance For Credit Losses Recovery</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_ProvisionForLoanAndLeaseLosses_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Provisions (credits)</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableAllowanceForCreditLosses_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/periodEndLabel">Ending balance</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FinancingReceivableAllowanceForCreditLossesIndividuallyEvaluatedForImpairment1" xlink:type="locator" xlink:label="us-gaap_FinancingReceivableAllowanceForCreditLossesIndividuallyEvaluatedForImpairment1" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableAllowanceForCreditLossesIndividuallyEvaluatedForImpairment1_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Ending balance: related to loans individually evaluated for impairment</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FinancingReceivableAllowanceForCreditLossesIndividuallyEvaluatedForImpairment1" xlink:to="us-gaap_FinancingReceivableAllowanceForCreditLossesIndividuallyEvaluatedForImpairment1_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableAllowanceForCreditLossesIndividuallyEvaluatedForImpairment1_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Financing Receivable Allowance For Credit Losses Individually Evaluated For Impairment1</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FinancingReceivableAllowanceForCreditLossesCollectivelyEvaluatedForImpairment" xlink:type="locator" xlink:label="us-gaap_FinancingReceivableAllowanceForCreditLossesCollectivelyEvaluatedForImpairment" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableAllowanceForCreditLossesCollectivelyEvaluatedForImpairment_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Ending balance: related to loans collectively evaluated for impairment</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FinancingReceivableAllowanceForCreditLossesCollectivelyEvaluatedForImpairment" xlink:to="us-gaap_FinancingReceivableAllowanceForCreditLossesCollectivelyEvaluatedForImpairment_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableAllowanceForCreditLossesCollectivelyEvaluatedForImpairment_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Financing Receivable Allowance For Credit Losses Collectively Evaluated For Impairment</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FinancingReceivableIndividuallyEvaluatedForImpairment" xlink:type="locator" xlink:label="us-gaap_FinancingReceivableIndividuallyEvaluatedForImpairment" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableIndividuallyEvaluatedForImpairment_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Ending balance: individually evaluated for impairment</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FinancingReceivableIndividuallyEvaluatedForImpairment" xlink:to="us-gaap_FinancingReceivableIndividuallyEvaluatedForImpairment_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableIndividuallyEvaluatedForImpairment_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Financing Receivable Individually Evaluated For Impairment</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FinancingReceivableCollectivelyEvaluatedForImpairment" xlink:type="locator" xlink:label="us-gaap_FinancingReceivableCollectivelyEvaluatedForImpairment" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableCollectivelyEvaluatedForImpairment_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Ending balance: collectively evaluated for impairment</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FinancingReceivableCollectivelyEvaluatedForImpairment" xlink:to="us-gaap_FinancingReceivableCollectivelyEvaluatedForImpairment_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableCollectivelyEvaluatedForImpairment_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Financing Receivable Collectively Evaluated For Impairment</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_AllowanceForLoanLossesOnBasisOfCalculationMethodologyAbstract" xlink:type="locator" xlink:label="pbhc_AllowanceForLoanLossesOnBasisOfCalculationMethodologyAbstract" />
    <link:label xml:lang="en-US" xlink:label="pbhc_AllowanceForLoanLossesOnBasisOfCalculationMethodologyAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Allowance for Loan Losses on Basis of Calculation Methodology [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_AllowanceForLoanLossesOnBasisOfCalculationMethodologyAbstract" xlink:to="pbhc_AllowanceForLoanLossesOnBasisOfCalculationMethodologyAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_AllowanceForLoanLossesOnBasisOfCalculationMethodologyAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Allowance For Loan Losses On Basis Of Calculation Methodology Abstract</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AllowanceForLoanLossesOnBasisOfCalculationMethodologyAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Allowance For Loan Losses On Basis Of Calculation Methodology [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_FinancingReceivableAllowanceForCreditLossesSpecificallyReserved_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Specifically reserved</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_FinancingReceivableAllowanceForCreditLossesSpecificallyReserved_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Financing Receivable Allowance For Credit Losses Specifically Reserved</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_FinancingReceivableAllowanceForCreditLossesHistoricalLossRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Historical loss rate</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_FinancingReceivableAllowanceForCreditLossesHistoricalLossRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Financing Receivable Allowance For Credit Losses Historical Loss Rate</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_FinancingReceivableAllowanceForCreditLossesQualitativeFactors_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Qualitative factors</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_FinancingReceivableAllowanceForCreditLossesQualitativeFactors_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Financing Receivable Allowance For Credit Losses Qualitative Factors</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_FinancingReceivableAllowanceForCreditLossesOther_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Other</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_FinancingReceivableAllowanceForCreditLossesOther_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Financing Receivable Allowance For Credit Losses Other</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancingReceivableAllowanceForCreditLosses_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Total</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ServicingAssetAtCarryingValueAmount" xlink:type="locator" xlink:label="pbhc_ServicingAssetAtCarryingValueAmount" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ServicingAssetAtCarryingValueAmount_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Carrying value as of the balance sheet date of unpaid principal balances of mortgage and other loans serviced for others.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ServicingAssetAtCarryingValueAmount" xlink:to="pbhc_ServicingAssetAtCarryingValueAmount_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_CapitalizedServicingAssetAtCarryingValueAmount" xlink:type="locator" xlink:label="pbhc_CapitalizedServicingAssetAtCarryingValueAmount" />
    <link:label xml:lang="en-US" xlink:label="pbhc_CapitalizedServicingAssetAtCarryingValueAmount_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Carrying value as of the balance sheet date of a capitalized servicing rights asset.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_CapitalizedServicingAssetAtCarryingValueAmount" xlink:to="pbhc_CapitalizedServicingAssetAtCarryingValueAmount_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ServicingAssetAtCarryingValueAmount_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Unpaid principal balance of mortgage and other loans</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ServicingAssetAtCarryingValueAmount_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Servicing Asset At Carrying Value Amount</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/srt/2020/elts/srt-2020-01-31.xsd#srt_MortgageLoansOnRealEstateNumberOfLoans" xlink:type="locator" xlink:label="srt_MortgageLoansOnRealEstateNumberOfLoans" />
    <link:label xml:lang="en-US" xlink:label="srt_MortgageLoansOnRealEstateNumberOfLoans_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Number of loans</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="srt_MortgageLoansOnRealEstateNumberOfLoans" xlink:to="srt_MortgageLoansOnRealEstateNumberOfLoans_lbl" />
    <link:label xml:lang="en-US" xlink:label="srt_MortgageLoansOnRealEstateNumberOfLoans_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Mortgage Loans On Real Estate Number Of Loans</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_CapitalizedServicingAssetAtCarryingValueAmount_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Capitalized servicing rights</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_CapitalizedServicingAssetAtCarryingValueAmount_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Capitalized Servicing Asset At Carrying Value Amount</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ServicingAssetAtAmortizedValueAdditions" xlink:type="locator" xlink:label="us-gaap_ServicingAssetAtAmortizedValueAdditions" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ServicingAssetAtAmortizedValueAdditions_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Mortgage servicing rights capitalized</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ServicingAssetAtAmortizedValueAdditions" xlink:to="us-gaap_ServicingAssetAtAmortizedValueAdditions_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ServicingAssetAtAmortizedValueAdditions_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Servicing Asset At Amortized Value Additions</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ServicingAssetAtAmortizedValueAmortization1" xlink:type="locator" xlink:label="us-gaap_ServicingAssetAtAmortizedValueAmortization1" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ServicingAssetAtAmortizedValueAmortization1_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Mortgage servicing rights amortized</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ServicingAssetAtAmortizedValueAmortization1" xlink:to="us-gaap_ServicingAssetAtAmortizedValueAmortization1_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ServicingAssetAtAmortizedValueAmortization1_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Servicing Asset At Amortized Value Amortization1</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ScheduleOfPropertyPlantAndEquipmentTable" xlink:type="locator" xlink:label="us-gaap_ScheduleOfPropertyPlantAndEquipmentTable" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfPropertyPlantAndEquipmentTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Schedule Of Property Plant And Equipment [Table]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ScheduleOfPropertyPlantAndEquipmentTable" xlink:to="us-gaap_ScheduleOfPropertyPlantAndEquipmentTable_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfPropertyPlantAndEquipmentTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Property Plant And Equipment [Table]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LandMember" xlink:type="locator" xlink:label="us-gaap_LandMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LandMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Land [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LandMember" xlink:to="us-gaap_LandMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LandMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Land [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_BuildingMember" xlink:type="locator" xlink:label="us-gaap_BuildingMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_BuildingMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Buildings [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_BuildingMember" xlink:to="us-gaap_BuildingMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_BuildingMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Building [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FurnitureAndFixturesMember" xlink:type="locator" xlink:label="us-gaap_FurnitureAndFixturesMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FurnitureAndFixturesMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Furniture, Fixtures and Equipment [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FurnitureAndFixturesMember" xlink:to="us-gaap_FurnitureAndFixturesMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FurnitureAndFixturesMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Furniture And Fixtures [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ConstructionInProgressMember" xlink:type="locator" xlink:label="us-gaap_ConstructionInProgressMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ConstructionInProgressMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Construction in Progress [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ConstructionInProgressMember" xlink:to="us-gaap_ConstructionInProgressMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ConstructionInProgressMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Construction In Progress [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_PropertyPlantAndEquipmentLineItems" xlink:type="locator" xlink:label="us-gaap_PropertyPlantAndEquipmentLineItems" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PropertyPlantAndEquipmentLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Premises and Equipment [Line Items]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_PropertyPlantAndEquipmentLineItems" xlink:to="us-gaap_PropertyPlantAndEquipmentLineItems_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PropertyPlantAndEquipmentLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Property Plant And Equipment [Line Items]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_PropertyPlantAndEquipmentGross" xlink:type="locator" xlink:label="us-gaap_PropertyPlantAndEquipmentGross" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PropertyPlantAndEquipmentGross_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Premises and equipment, gross</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_PropertyPlantAndEquipmentGross" xlink:to="us-gaap_PropertyPlantAndEquipmentGross_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PropertyPlantAndEquipmentGross_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Property Plant And Equipment Gross</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AccumulatedDepreciationDepletionAndAmortizationPropertyPlantAndEquipment" xlink:type="locator" xlink:label="us-gaap_AccumulatedDepreciationDepletionAndAmortizationPropertyPlantAndEquipment" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AccumulatedDepreciationDepletionAndAmortizationPropertyPlantAndEquipment_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Less: Accumulated depreciation</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AccumulatedDepreciationDepletionAndAmortizationPropertyPlantAndEquipment" xlink:to="us-gaap_AccumulatedDepreciationDepletionAndAmortizationPropertyPlantAndEquipment_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AccumulatedDepreciationDepletionAndAmortizationPropertyPlantAndEquipment_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Accumulated Depreciation Depletion And Amortization Property Plant And Equipment</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_PropertyPlantAndEquipmentNet_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Premises and equipment, net</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_Depreciation_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Depreciation expense</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_MortgageLoansInProcessOfForeclosureAmount" xlink:type="locator" xlink:label="us-gaap_MortgageLoansInProcessOfForeclosureAmount" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_MortgageLoansInProcessOfForeclosureAmount_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Residential real estate loans in the process of foreclosure</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_MortgageLoansInProcessOfForeclosureAmount" xlink:to="us-gaap_MortgageLoansInProcessOfForeclosureAmount_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_MortgageLoansInProcessOfForeclosureAmount_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Mortgage Loans In Process Of Foreclosure Amount</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_NumberOfRealEstateProperties" xlink:type="locator" xlink:label="us-gaap_NumberOfRealEstateProperties" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_NumberOfRealEstateProperties_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Number of properties</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_NumberOfRealEstateProperties" xlink:to="us-gaap_NumberOfRealEstateProperties_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_NumberOfRealEstateProperties_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Number Of Real Estate Properties</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ScheduleOfBusinessAcquisitionsByAcquisitionTable" xlink:type="locator" xlink:label="us-gaap_ScheduleOfBusinessAcquisitionsByAcquisitionTable" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfBusinessAcquisitionsByAcquisitionTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Schedule Of Business Acquisitions By Acquisition [Table]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ScheduleOfBusinessAcquisitionsByAcquisitionTable" xlink:to="us-gaap_ScheduleOfBusinessAcquisitionsByAcquisitionTable_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfBusinessAcquisitionsByAcquisitionTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Business Acquisitions By Acquisition [Table]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_BusinessAcquisitionAxis" xlink:type="locator" xlink:label="us-gaap_BusinessAcquisitionAxis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_BusinessAcquisitionAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Business Acquisition</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_BusinessAcquisitionAxis" xlink:to="us-gaap_BusinessAcquisitionAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_BusinessAcquisitionAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Business Acquisition [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_BusinessAcquisitionAcquireeDomain" xlink:type="locator" xlink:label="us-gaap_BusinessAcquisitionAcquireeDomain" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_BusinessAcquisitionAcquireeDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Business Acquisition, Acquiree</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_BusinessAcquisitionAcquireeDomain" xlink:to="us-gaap_BusinessAcquisitionAcquireeDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_BusinessAcquisitionAcquireeDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Business Acquisition Acquiree [Domain]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_AcquiredSubsidiaryOneMember" xlink:type="locator" xlink:label="pbhc_AcquiredSubsidiaryOneMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_AcquiredSubsidiaryOneMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The company that is being acquired or purchased in a merger or acquisition. The acquiree is also known as the "target firm".</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_AcquiredSubsidiaryOneMember" xlink:to="pbhc_AcquiredSubsidiaryOneMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_AcquiredSubsidiaryOneMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Branches [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AcquiredSubsidiaryOneMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Acquired Subsidiary One [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_AcquiredSubsidiaryTwoMember" xlink:type="locator" xlink:label="pbhc_AcquiredSubsidiaryTwoMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_AcquiredSubsidiaryTwoMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The company that is being acquired or purchased in a merger or acquisition. The acquiree is also known as the "target firm".</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_AcquiredSubsidiaryTwoMember" xlink:to="pbhc_AcquiredSubsidiaryTwoMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_AcquiredSubsidiaryTwoMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Fitzgibbons Agency [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AcquiredSubsidiaryTwoMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Acquired Subsidiary Two [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_BusinessAcquisitionLineItems" xlink:type="locator" xlink:label="us-gaap_BusinessAcquisitionLineItems" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_BusinessAcquisitionLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Business Acquisition [Line Items]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_BusinessAcquisitionLineItems" xlink:to="us-gaap_BusinessAcquisitionLineItems_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_BusinessAcquisitionLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Business Acquisition [Line Items]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_GoodwillAcquiredDuringPeriod" xlink:type="locator" xlink:label="us-gaap_GoodwillAcquiredDuringPeriod" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_GoodwillAcquiredDuringPeriod_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Goodwill acquired</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_GoodwillAcquiredDuringPeriod" xlink:to="us-gaap_GoodwillAcquiredDuringPeriod_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_GoodwillAcquiredDuringPeriod_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Goodwill Acquired During Period</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FinitelivedIntangibleAssetsAcquired1" xlink:type="locator" xlink:label="us-gaap_FinitelivedIntangibleAssetsAcquired1" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinitelivedIntangibleAssetsAcquired1_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Identifiable intangible asset</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FinitelivedIntangibleAssetsAcquired1" xlink:to="us-gaap_FinitelivedIntangibleAssetsAcquired1_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinitelivedIntangibleAssetsAcquired1_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Finitelived Intangible Assets Acquired1</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AcquiredFiniteLivedIntangibleAssetsWeightedAverageUsefulLife" xlink:type="locator" xlink:label="us-gaap_AcquiredFiniteLivedIntangibleAssetsWeightedAverageUsefulLife" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AcquiredFiniteLivedIntangibleAssetsWeightedAverageUsefulLife_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Weighted average remaining amortization period of intangible asset</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AcquiredFiniteLivedIntangibleAssetsWeightedAverageUsefulLife" xlink:to="us-gaap_AcquiredFiniteLivedIntangibleAssetsWeightedAverageUsefulLife_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AcquiredFiniteLivedIntangibleAssetsWeightedAverageUsefulLife_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Acquired Finite Lived Intangible Assets Weighted Average Useful Life</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FiniteLivedIntangibleAssetsNetAbstract" xlink:type="locator" xlink:label="us-gaap_FiniteLivedIntangibleAssetsNetAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FiniteLivedIntangibleAssetsNetAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Gross carrying amount and accumulated amortization for identifiable intangible asset [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FiniteLivedIntangibleAssetsNetAbstract" xlink:to="us-gaap_FiniteLivedIntangibleAssetsNetAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FiniteLivedIntangibleAssetsNetAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Finite Lived Intangible Assets Net [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FiniteLivedIntangibleAssetsGross" xlink:type="locator" xlink:label="us-gaap_FiniteLivedIntangibleAssetsGross" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FiniteLivedIntangibleAssetsGross_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Gross carrying amount</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FiniteLivedIntangibleAssetsGross" xlink:to="us-gaap_FiniteLivedIntangibleAssetsGross_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FiniteLivedIntangibleAssetsGross_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Finite Lived Intangible Assets Gross</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FiniteLivedIntangibleAssetsAccumulatedAmortization" xlink:type="locator" xlink:label="us-gaap_FiniteLivedIntangibleAssetsAccumulatedAmortization" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FiniteLivedIntangibleAssetsAccumulatedAmortization_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Accumulated amortization</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FiniteLivedIntangibleAssetsAccumulatedAmortization" xlink:to="us-gaap_FiniteLivedIntangibleAssetsAccumulatedAmortization_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FiniteLivedIntangibleAssetsAccumulatedAmortization_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Finite Lived Intangible Assets Accumulated Amortization</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FiniteLivedIntangibleAssetsNet" xlink:type="locator" xlink:label="us-gaap_FiniteLivedIntangibleAssetsNet" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FiniteLivedIntangibleAssetsNet_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Net amortizing intangibles</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FiniteLivedIntangibleAssetsNet" xlink:to="us-gaap_FiniteLivedIntangibleAssetsNet_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FiniteLivedIntangibleAssetsNet_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Finite Lived Intangible Assets Net</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FiniteLivedIntangibleAssetsFutureAmortizationExpenseAbstract" xlink:type="locator" xlink:label="us-gaap_FiniteLivedIntangibleAssetsFutureAmortizationExpenseAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FiniteLivedIntangibleAssetsFutureAmortizationExpenseAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Estimated amortization expense for each of the five succeeding years [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FiniteLivedIntangibleAssetsFutureAmortizationExpenseAbstract" xlink:to="us-gaap_FiniteLivedIntangibleAssetsFutureAmortizationExpenseAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FiniteLivedIntangibleAssetsFutureAmortizationExpenseAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Finite Lived Intangible Assets Future Amortization Expense [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FiniteLivedIntangibleAssetsAmortizationExpenseNextTwelveMonths" xlink:type="locator" xlink:label="us-gaap_FiniteLivedIntangibleAssetsAmortizationExpenseNextTwelveMonths" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FiniteLivedIntangibleAssetsAmortizationExpenseNextTwelveMonths_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">2021</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FiniteLivedIntangibleAssetsAmortizationExpenseNextTwelveMonths" xlink:to="us-gaap_FiniteLivedIntangibleAssetsAmortizationExpenseNextTwelveMonths_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FiniteLivedIntangibleAssetsAmortizationExpenseNextTwelveMonths_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Finite Lived Intangible Assets Amortization Expense Next Twelve Months</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FiniteLivedIntangibleAssetsAmortizationExpenseYearTwo" xlink:type="locator" xlink:label="us-gaap_FiniteLivedIntangibleAssetsAmortizationExpenseYearTwo" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FiniteLivedIntangibleAssetsAmortizationExpenseYearTwo_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">2022</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FiniteLivedIntangibleAssetsAmortizationExpenseYearTwo" xlink:to="us-gaap_FiniteLivedIntangibleAssetsAmortizationExpenseYearTwo_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FiniteLivedIntangibleAssetsAmortizationExpenseYearTwo_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Finite Lived Intangible Assets Amortization Expense Year Two</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FiniteLivedIntangibleAssetsAmortizationExpenseYearThree" xlink:type="locator" xlink:label="us-gaap_FiniteLivedIntangibleAssetsAmortizationExpenseYearThree" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FiniteLivedIntangibleAssetsAmortizationExpenseYearThree_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">2023</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FiniteLivedIntangibleAssetsAmortizationExpenseYearThree" xlink:to="us-gaap_FiniteLivedIntangibleAssetsAmortizationExpenseYearThree_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FiniteLivedIntangibleAssetsAmortizationExpenseYearThree_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Finite Lived Intangible Assets Amortization Expense Year Three</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FiniteLivedIntangibleAssetsAmortizationExpenseYearFour" xlink:type="locator" xlink:label="us-gaap_FiniteLivedIntangibleAssetsAmortizationExpenseYearFour" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FiniteLivedIntangibleAssetsAmortizationExpenseYearFour_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">2024</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FiniteLivedIntangibleAssetsAmortizationExpenseYearFour" xlink:to="us-gaap_FiniteLivedIntangibleAssetsAmortizationExpenseYearFour_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FiniteLivedIntangibleAssetsAmortizationExpenseYearFour_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Finite Lived Intangible Assets Amortization Expense Year Four</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FiniteLivedIntangibleAssetsAmortizationExpenseYearFive" xlink:type="locator" xlink:label="us-gaap_FiniteLivedIntangibleAssetsAmortizationExpenseYearFive" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FiniteLivedIntangibleAssetsAmortizationExpenseYearFive_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">2025</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FiniteLivedIntangibleAssetsAmortizationExpenseYearFive" xlink:to="us-gaap_FiniteLivedIntangibleAssetsAmortizationExpenseYearFive_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FiniteLivedIntangibleAssetsAmortizationExpenseYearFive_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Finite Lived Intangible Assets Amortization Expense Year Five</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FiniteLivedIntangibleAssetsAmortizationExpenseAfterYearFive" xlink:type="locator" xlink:label="us-gaap_FiniteLivedIntangibleAssetsAmortizationExpenseAfterYearFive" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FiniteLivedIntangibleAssetsAmortizationExpenseAfterYearFive_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Thereafter</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FiniteLivedIntangibleAssetsAmortizationExpenseAfterYearFive" xlink:to="us-gaap_FiniteLivedIntangibleAssetsAmortizationExpenseAfterYearFive_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FiniteLivedIntangibleAssetsAmortizationExpenseAfterYearFive_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Finite Lived Intangible Assets Amortization Expense After Year Five</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DepositsByTypeAbstract" xlink:type="locator" xlink:label="us-gaap_DepositsByTypeAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DepositsByTypeAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Summary of deposits [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DepositsByTypeAbstract" xlink:to="us-gaap_DepositsByTypeAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DepositsByTypeAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deposits By Type [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DepositsSavingsDeposits" xlink:type="locator" xlink:label="us-gaap_DepositsSavingsDeposits" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DepositsSavingsDeposits_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Savings accounts</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DepositsSavingsDeposits" xlink:to="us-gaap_DepositsSavingsDeposits_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DepositsSavingsDeposits_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deposits Savings Deposits</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_TimeDepositsLessThan100000" xlink:type="locator" xlink:label="us-gaap_TimeDepositsLessThan100000" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TimeDepositsLessThan100000_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Time accounts</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_TimeDepositsLessThan100000" xlink:to="us-gaap_TimeDepositsLessThan100000_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TimeDepositsLessThan100000_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Time Deposits Less Than100000</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_TimeDeposits250000OrMore_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Time accounts of $250,000 or more</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_TimeDeposits250000OrMore_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Time Deposits250000 Or More</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DepositsMoneyManagementAccounts_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Money management accounts</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DepositsMoneyManagementAccounts_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deposits Money Management Accounts</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DepositsMoneyMarketDeposits" xlink:type="locator" xlink:label="us-gaap_DepositsMoneyMarketDeposits" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DepositsMoneyMarketDeposits_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">MMDA accounts</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DepositsMoneyMarketDeposits" xlink:to="us-gaap_DepositsMoneyMarketDeposits_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DepositsMoneyMarketDeposits_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deposits Money Market Deposits</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DemandDepositAccounts" xlink:type="locator" xlink:label="us-gaap_DemandDepositAccounts" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DemandDepositAccounts_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Demand deposit interest-bearing</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DemandDepositAccounts" xlink:to="us-gaap_DemandDepositAccounts_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DemandDepositAccounts_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Demand Deposit Accounts</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_NoninterestBearingDepositLiabilities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Demand deposit noninterest-bearing</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MortgageEscrowFunds_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Mortgage escrow funds</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MortgageEscrowFunds_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Mortgage Escrow Funds</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_MaturitiesOfTimeDepositsAbstract" xlink:type="locator" xlink:label="us-gaap_MaturitiesOfTimeDepositsAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_MaturitiesOfTimeDepositsAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Maturities of time deposits [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_MaturitiesOfTimeDepositsAbstract" xlink:to="us-gaap_MaturitiesOfTimeDepositsAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_MaturitiesOfTimeDepositsAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Maturities Of Time Deposits [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_TimeDepositMaturitiesYearOne" xlink:type="locator" xlink:label="us-gaap_TimeDepositMaturitiesYearOne" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TimeDepositMaturitiesYearOne_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">2021</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_TimeDepositMaturitiesYearOne" xlink:to="us-gaap_TimeDepositMaturitiesYearOne_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TimeDepositMaturitiesYearOne_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Time Deposit Maturities Year One</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_TimeDepositMaturitiesYearTwo" xlink:type="locator" xlink:label="us-gaap_TimeDepositMaturitiesYearTwo" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TimeDepositMaturitiesYearTwo_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">2022</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_TimeDepositMaturitiesYearTwo" xlink:to="us-gaap_TimeDepositMaturitiesYearTwo_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TimeDepositMaturitiesYearTwo_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Time Deposit Maturities Year Two</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_TimeDepositMaturitiesYearThree" xlink:type="locator" xlink:label="us-gaap_TimeDepositMaturitiesYearThree" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TimeDepositMaturitiesYearThree_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">2023</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_TimeDepositMaturitiesYearThree" xlink:to="us-gaap_TimeDepositMaturitiesYearThree_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TimeDepositMaturitiesYearThree_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Time Deposit Maturities Year Three</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_TimeDepositMaturitiesYearFour" xlink:type="locator" xlink:label="us-gaap_TimeDepositMaturitiesYearFour" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TimeDepositMaturitiesYearFour_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">2024</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_TimeDepositMaturitiesYearFour" xlink:to="us-gaap_TimeDepositMaturitiesYearFour_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TimeDepositMaturitiesYearFour_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Time Deposit Maturities Year Four</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_TimeDepositMaturitiesYearFive" xlink:type="locator" xlink:label="us-gaap_TimeDepositMaturitiesYearFive" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TimeDepositMaturitiesYearFive_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">2025</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_TimeDepositMaturitiesYearFive" xlink:to="us-gaap_TimeDepositMaturitiesYearFive_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TimeDepositMaturitiesYearFive_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Time Deposit Maturities Year Five</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_TimeDepositMaturitiesAfterYearFive" xlink:type="locator" xlink:label="us-gaap_TimeDepositMaturitiesAfterYearFive" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TimeDepositMaturitiesAfterYearFive_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Thereafter</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_TimeDepositMaturitiesAfterYearFive" xlink:to="us-gaap_TimeDepositMaturitiesAfterYearFive_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TimeDepositMaturitiesAfterYearFive_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Time Deposit Maturities After Year Five</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_TimeDeposits" xlink:type="locator" xlink:label="us-gaap_TimeDeposits" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TimeDeposits_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Total</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_TimeDeposits" xlink:to="us-gaap_TimeDeposits_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TimeDeposits_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Time Deposits</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_DepositsLineItems" xlink:type="locator" xlink:label="pbhc_DepositsLineItems" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DepositsLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Line items represent financial concepts included in a table. These concepts are used to disclose reportable information associated with domain members defined in one or many axes to the table.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_DepositsLineItems" xlink:to="pbhc_DepositsLineItems_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_DepositsTable" xlink:type="locator" xlink:label="pbhc_DepositsTable" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DepositsTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Disclosure of information about all deposit liabilities held by the entity.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_DepositsTable" xlink:to="pbhc_DepositsTable_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DepositsTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Deposits [Table]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DepositsTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deposits [Table]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_StatementBusinessSegmentsAxis" xlink:type="locator" xlink:label="us-gaap_StatementBusinessSegmentsAxis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_StatementBusinessSegmentsAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Segments</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_StatementBusinessSegmentsAxis" xlink:to="us-gaap_StatementBusinessSegmentsAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_StatementBusinessSegmentsAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Statement Business Segments [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_SegmentDomain" xlink:type="locator" xlink:label="us-gaap_SegmentDomain" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SegmentDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Segments</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_SegmentDomain" xlink:to="us-gaap_SegmentDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SegmentDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Segment [Domain]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_NonBrokeredDepositMember" xlink:type="locator" xlink:label="pbhc_NonBrokeredDepositMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NonBrokeredDepositMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Refers to the non-brokered deposits.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_NonBrokeredDepositMember" xlink:to="pbhc_NonBrokeredDepositMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NonBrokeredDepositMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Non-Brokered [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NonBrokeredDepositMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Non Brokered Deposit [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_BrokeredDepositMember" xlink:type="locator" xlink:label="pbhc_BrokeredDepositMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_BrokeredDepositMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">A large-denomination bank deposit that is sold by a bank to a brokerage, which then divides it into smaller pieces for sale to its customers, for example core deposits.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_BrokeredDepositMember" xlink:to="pbhc_BrokeredDepositMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_BrokeredDepositMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Brokered [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_BrokeredDepositMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Brokered Deposit [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DepositsLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Deposits [Line Items]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DepositsLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deposits [Line Items]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DebtInstrumentTable" xlink:type="locator" xlink:label="us-gaap_DebtInstrumentTable" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtInstrumentTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Debt Instrument [Table]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DebtInstrumentTable" xlink:to="us-gaap_DebtInstrumentTable_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtInstrumentTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Debt Instrument [Table]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FederalHomeLoanBankAdvancesMember" xlink:type="locator" xlink:label="us-gaap_FederalHomeLoanBankAdvancesMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FederalHomeLoanBankAdvancesMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">FHLB Advances [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FederalHomeLoanBankAdvancesMember" xlink:to="us-gaap_FederalHomeLoanBankAdvancesMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FederalHomeLoanBankAdvancesMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Federal Home Loan Bank Advances [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DebtInstrumentLineItems" xlink:type="locator" xlink:label="us-gaap_DebtInstrumentLineItems" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtInstrumentLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Debt Instrument [Line Items]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DebtInstrumentLineItems" xlink:to="us-gaap_DebtInstrumentLineItems_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtInstrumentLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Debt Instrument [Line Items]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ShortTermBorrowingsAbstract" xlink:type="locator" xlink:label="us-gaap_ShortTermBorrowingsAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShortTermBorrowingsAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Short-term [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ShortTermBorrowingsAbstract" xlink:to="us-gaap_ShortTermBorrowingsAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShortTermBorrowingsAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Short Term Borrowings [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShortTermBorrowings_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Total short-term borrowings</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LongTermDebtAbstract" xlink:type="locator" xlink:label="us-gaap_LongTermDebtAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LongTermDebtAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Long-term [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LongTermDebtAbstract" xlink:to="us-gaap_LongTermDebtAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LongTermDebtAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Long Term Debt [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_LongTermDebtNoncurrent_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Total long-term borrowings</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FederalHomeLoanBankAdvancesMaturitiesSummaryAbstract" xlink:type="locator" xlink:label="us-gaap_FederalHomeLoanBankAdvancesMaturitiesSummaryAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FederalHomeLoanBankAdvancesMaturitiesSummaryAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Advances with FHLB [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FederalHomeLoanBankAdvancesMaturitiesSummaryAbstract" xlink:to="us-gaap_FederalHomeLoanBankAdvancesMaturitiesSummaryAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FederalHomeLoanBankAdvancesMaturitiesSummaryAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Federal Home Loan Bank Advances Maturities Summary [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FederalHomeLoanBankAdvancesMaturitiesSummaryDueWithinOneYearOfBalanceSheetDate" xlink:type="locator" xlink:label="us-gaap_FederalHomeLoanBankAdvancesMaturitiesSummaryDueWithinOneYearOfBalanceSheetDate" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FederalHomeLoanBankAdvancesMaturitiesSummaryDueWithinOneYearOfBalanceSheetDate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Due within 1 year</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FederalHomeLoanBankAdvancesMaturitiesSummaryDueWithinOneYearOfBalanceSheetDate" xlink:to="us-gaap_FederalHomeLoanBankAdvancesMaturitiesSummaryDueWithinOneYearOfBalanceSheetDate_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FederalHomeLoanBankAdvancesMaturitiesSummaryDueWithinOneYearOfBalanceSheetDate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Federal Home Loan Bank Advances Maturities Summary Due Within One Year Of Balance Sheet Date</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FederalHomeLoanBankAdvancesMaturitiesSummaryDueFromOneToTwoYearsOfBalanceSheetDate" xlink:type="locator" xlink:label="us-gaap_FederalHomeLoanBankAdvancesMaturitiesSummaryDueFromOneToTwoYearsOfBalanceSheetDate" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FederalHomeLoanBankAdvancesMaturitiesSummaryDueFromOneToTwoYearsOfBalanceSheetDate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Due within 2 years</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FederalHomeLoanBankAdvancesMaturitiesSummaryDueFromOneToTwoYearsOfBalanceSheetDate" xlink:to="us-gaap_FederalHomeLoanBankAdvancesMaturitiesSummaryDueFromOneToTwoYearsOfBalanceSheetDate_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FederalHomeLoanBankAdvancesMaturitiesSummaryDueFromOneToTwoYearsOfBalanceSheetDate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Federal Home Loan Bank Advances Maturities Summary Due From One To Two Years Of Balance Sheet Date</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FederalHomeLoanBankAdvancesMaturitiesSummaryDueAfterFiveYearsOfBalanceSheetDate" xlink:type="locator" xlink:label="us-gaap_FederalHomeLoanBankAdvancesMaturitiesSummaryDueAfterFiveYearsOfBalanceSheetDate" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FederalHomeLoanBankAdvancesMaturitiesSummaryDueAfterFiveYearsOfBalanceSheetDate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Due within 10 years</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FederalHomeLoanBankAdvancesMaturitiesSummaryDueAfterFiveYearsOfBalanceSheetDate" xlink:to="us-gaap_FederalHomeLoanBankAdvancesMaturitiesSummaryDueAfterFiveYearsOfBalanceSheetDate_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FederalHomeLoanBankAdvancesMaturitiesSummaryDueAfterFiveYearsOfBalanceSheetDate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Federal Home Loan Bank Advances Maturities Summary Due After Five Years Of Balance Sheet Date</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AdvancesFromFederalHomeLoanBanks" xlink:type="locator" xlink:label="us-gaap_AdvancesFromFederalHomeLoanBanks" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AdvancesFromFederalHomeLoanBanks_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Total advances with FHLB</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AdvancesFromFederalHomeLoanBanks" xlink:to="us-gaap_AdvancesFromFederalHomeLoanBanks_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AdvancesFromFederalHomeLoanBanks_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Advances From Federal Home Loan Banks</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_LongTermDebtNoncurrent_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Total long-term fixed rate borrowings</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_FederalHomeLoanBankAdvancesMaturitiesSummaryByInterestRateTypeUnderOneYearInterestRate" xlink:type="locator" xlink:label="pbhc_FederalHomeLoanBankAdvancesMaturitiesSummaryByInterestRateTypeUnderOneYearInterestRate" />
    <link:label xml:lang="en-US" xlink:label="pbhc_FederalHomeLoanBankAdvancesMaturitiesSummaryByInterestRateTypeUnderOneYearInterestRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Represents the percentage of interest rates from lowest to highest that are applicable to Federal Home Loan Bank advances due within one year of the Balance Sheet date.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_FederalHomeLoanBankAdvancesMaturitiesSummaryByInterestRateTypeUnderOneYearInterestRate" xlink:to="pbhc_FederalHomeLoanBankAdvancesMaturitiesSummaryByInterestRateTypeUnderOneYearInterestRate_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_FederalHomeLoanBankAdvancesMaturitiesSummaryByInterestRateTypeUnderTwoYearInterestRate" xlink:type="locator" xlink:label="pbhc_FederalHomeLoanBankAdvancesMaturitiesSummaryByInterestRateTypeUnderTwoYearInterestRate" />
    <link:label xml:lang="en-US" xlink:label="pbhc_FederalHomeLoanBankAdvancesMaturitiesSummaryByInterestRateTypeUnderTwoYearInterestRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Represents the range of interest rates from lowest to highest that are applicable to Federal Home Loan Bank advances due within two years of the Balance Sheet date.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_FederalHomeLoanBankAdvancesMaturitiesSummaryByInterestRateTypeUnderTwoYearInterestRate" xlink:to="pbhc_FederalHomeLoanBankAdvancesMaturitiesSummaryByInterestRateTypeUnderTwoYearInterestRate_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_FederalHomeLoanBankAdvancesMaturitiesSummaryByInterestRateWithinTenYearsInterestRate" xlink:type="locator" xlink:label="pbhc_FederalHomeLoanBankAdvancesMaturitiesSummaryByInterestRateWithinTenYearsInterestRate" />
    <link:label xml:lang="en-US" xlink:label="pbhc_FederalHomeLoanBankAdvancesMaturitiesSummaryByInterestRateWithinTenYearsInterestRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Represents the interest rate applicable to Federal Home Loan Bank advances due within ten years of the Balance Sheet date.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_FederalHomeLoanBankAdvancesMaturitiesSummaryByInterestRateWithinTenYearsInterestRate" xlink:to="pbhc_FederalHomeLoanBankAdvancesMaturitiesSummaryByInterestRateWithinTenYearsInterestRate_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FederalHomeLoanBankAdvancesMaturitiesSummaryByInterestRateTypeByRateTypeAbstract" xlink:type="locator" xlink:label="us-gaap_FederalHomeLoanBankAdvancesMaturitiesSummaryByInterestRateTypeByRateTypeAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FederalHomeLoanBankAdvancesMaturitiesSummaryByInterestRateTypeByRateTypeAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Federal Home Loan Bank advances interest rate [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FederalHomeLoanBankAdvancesMaturitiesSummaryByInterestRateTypeByRateTypeAbstract" xlink:to="us-gaap_FederalHomeLoanBankAdvancesMaturitiesSummaryByInterestRateTypeByRateTypeAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FederalHomeLoanBankAdvancesMaturitiesSummaryByInterestRateTypeByRateTypeAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Federal Home Loan Bank Advances Maturities Summary By Interest Rate Type By Rate Type [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_FederalHomeLoanBankAdvancesMaturitiesSummaryByInterestRateTypeUnderOneYearInterestRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Due within 1 year interest rate</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_FederalHomeLoanBankAdvancesMaturitiesSummaryByInterestRateTypeUnderOneYearInterestRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Federal Home Loan Bank Advances Maturities Summary By Interest Rate Type Under One Year Interest Rate</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_FederalHomeLoanBankAdvancesMaturitiesSummaryByInterestRateTypeUnderTwoYearInterestRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Due within 2 years interest rate</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_FederalHomeLoanBankAdvancesMaturitiesSummaryByInterestRateTypeUnderTwoYearInterestRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Federal Home Loan Bank Advances Maturities Summary By Interest Rate Type Under Two Year Interest Rate</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_FederalHomeLoanBankAdvancesMaturitiesSummaryByInterestRateWithinTenYearsInterestRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Due within 10 years interest rate</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_FederalHomeLoanBankAdvancesMaturitiesSummaryByInterestRateWithinTenYearsInterestRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Federal Home Loan Bank Advances Maturities Summary By Interest Rate Within Ten Years Interest Rate</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LongTermDebtByMaturityAbstract" xlink:type="locator" xlink:label="us-gaap_LongTermDebtByMaturityAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LongTermDebtByMaturityAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Long-term Debt, Fiscal Year Maturity [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LongTermDebtByMaturityAbstract" xlink:to="us-gaap_LongTermDebtByMaturityAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LongTermDebtByMaturityAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Long Term Debt By Maturity [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInNextTwelveMonths" xlink:type="locator" xlink:label="us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInNextTwelveMonths" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInNextTwelveMonths_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">2021</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInNextTwelveMonths" xlink:to="us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInNextTwelveMonths_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInNextTwelveMonths_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Long Term Debt Maturities Repayments Of Principal In Next Twelve Months</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInYearTwo" xlink:type="locator" xlink:label="us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInYearTwo" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInYearTwo_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">2022</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInYearTwo" xlink:to="us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInYearTwo_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInYearTwo_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Long Term Debt Maturities Repayments Of Principal In Year Two</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInYearThree" xlink:type="locator" xlink:label="us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInYearThree" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInYearThree_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">2023</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInYearThree" xlink:to="us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInYearThree_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInYearThree_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Long Term Debt Maturities Repayments Of Principal In Year Three</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInYearFour" xlink:type="locator" xlink:label="us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInYearFour" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInYearFour_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">2024</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInYearFour" xlink:to="us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInYearFour_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInYearFour_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Long Term Debt Maturities Repayments Of Principal In Year Four</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInYearFive" xlink:type="locator" xlink:label="us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInYearFive" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInYearFive_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">2025</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInYearFive" xlink:to="us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInYearFive_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInYearFive_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Long Term Debt Maturities Repayments Of Principal In Year Five</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_CarryingValueOfResidentialMortgageLoansPledgedUnderBlanketCollateralAgreement" xlink:type="locator" xlink:label="pbhc_CarryingValueOfResidentialMortgageLoansPledgedUnderBlanketCollateralAgreement" />
    <link:label xml:lang="en-US" xlink:label="pbhc_CarryingValueOfResidentialMortgageLoansPledgedUnderBlanketCollateralAgreement_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The carrying value of collateral requirement associated with residential mortgage loans.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_CarryingValueOfResidentialMortgageLoansPledgedUnderBlanketCollateralAgreement" xlink:to="pbhc_CarryingValueOfResidentialMortgageLoansPledgedUnderBlanketCollateralAgreement_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_CarryingValueOfFhlbStockPledgedUnderBlanketCollateralAgreement" xlink:type="locator" xlink:label="pbhc_CarryingValueOfFhlbStockPledgedUnderBlanketCollateralAgreement" />
    <link:label xml:lang="en-US" xlink:label="pbhc_CarryingValueOfFhlbStockPledgedUnderBlanketCollateralAgreement_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The carrying value of collateral requirement associated with FHLB Stock.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_CarryingValueOfFhlbStockPledgedUnderBlanketCollateralAgreement" xlink:to="pbhc_CarryingValueOfFhlbStockPledgedUnderBlanketCollateralAgreement_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_NumberOfCorrespondingBanksWithLineOfCreditAvailable" xlink:type="locator" xlink:label="pbhc_NumberOfCorrespondingBanksWithLineOfCreditAvailable" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfCorrespondingBanksWithLineOfCreditAvailable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The number of corresponding banks line of credit facilities.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_NumberOfCorrespondingBanksWithLineOfCreditAvailable" xlink:to="pbhc_NumberOfCorrespondingBanksWithLineOfCreditAvailable_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_LineOfCreditFacilityMaximumBorrowingCapacityUnsecuredBasis" xlink:type="locator" xlink:label="pbhc_LineOfCreditFacilityMaximumBorrowingCapacityUnsecuredBasis" />
    <link:label xml:lang="en-US" xlink:label="pbhc_LineOfCreditFacilityMaximumBorrowingCapacityUnsecuredBasis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Maximum borrowing capacity on an unsecured basis, under the credit facility without consideration of any current restrictions on the amount that could be borrowed or the amounts currently outstanding under the facility.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_LineOfCreditFacilityMaximumBorrowingCapacityUnsecuredBasis" xlink:to="pbhc_LineOfCreditFacilityMaximumBorrowingCapacityUnsecuredBasis_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_LineOfCreditFacilityMaximumBorrowingCapacityRequiringCollateral" xlink:type="locator" xlink:label="pbhc_LineOfCreditFacilityMaximumBorrowingCapacityRequiringCollateral" />
    <link:label xml:lang="en-US" xlink:label="pbhc_LineOfCreditFacilityMaximumBorrowingCapacityRequiringCollateral_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Maximum borrowing capacity requiring collateral, under the credit facility without consideration of any current restrictions on the amount that could be borrowed or the amounts currently outstanding under the facility.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_LineOfCreditFacilityMaximumBorrowingCapacityRequiringCollateral" xlink:to="pbhc_LineOfCreditFacilityMaximumBorrowingCapacityRequiringCollateral_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LineOfCreditFacilityTable" xlink:type="locator" xlink:label="us-gaap_LineOfCreditFacilityTable" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LineOfCreditFacilityTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Line Of Credit Facility [Table]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LineOfCreditFacilityTable" xlink:to="us-gaap_LineOfCreditFacilityTable_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LineOfCreditFacilityTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Line Of Credit Facility [Table]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LineOfCreditFacilityAxis" xlink:type="locator" xlink:label="us-gaap_LineOfCreditFacilityAxis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LineOfCreditFacilityAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Lender Name</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LineOfCreditFacilityAxis" xlink:to="us-gaap_LineOfCreditFacilityAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LineOfCreditFacilityAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Line Of Credit Facility [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LineOfCreditFacilityLenderDomain" xlink:type="locator" xlink:label="us-gaap_LineOfCreditFacilityLenderDomain" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LineOfCreditFacilityLenderDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Line of Credit Facility, Lender</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LineOfCreditFacilityLenderDomain" xlink:to="us-gaap_LineOfCreditFacilityLenderDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LineOfCreditFacilityLenderDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Line Of Credit Facility Lender [Domain]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_FederalReserveBankOfNewYorkMember" xlink:type="locator" xlink:label="pbhc_FederalReserveBankOfNewYorkMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_FederalReserveBankOfNewYorkMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Borrowings from the Federal Reserve Bank of New York, which are primarily used to cover shortages in the required reserve balance and also in times of liquidity shortages.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_FederalReserveBankOfNewYorkMember" xlink:to="pbhc_FederalReserveBankOfNewYorkMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_FederalReserveBankOfNewYorkMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Federal Reserve Bank of New York [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_FederalReserveBankOfNewYorkMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Federal Reserve Bank Of New York [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_OtherCorrespondingBanksMember" xlink:type="locator" xlink:label="pbhc_OtherCorrespondingBanksMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_OtherCorrespondingBanksMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The other corresponding banks.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_OtherCorrespondingBanksMember" xlink:to="pbhc_OtherCorrespondingBanksMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_OtherCorrespondingBanksMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Other Corresponding Banks [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_OtherCorrespondingBanksMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Other Corresponding Banks [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_CreditFacilityAxis" xlink:type="locator" xlink:label="us-gaap_CreditFacilityAxis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CreditFacilityAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Credit Facility</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_CreditFacilityAxis" xlink:to="us-gaap_CreditFacilityAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CreditFacilityAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Credit Facility [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_CreditFacilityDomain" xlink:type="locator" xlink:label="us-gaap_CreditFacilityDomain" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CreditFacilityDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Credit Facility</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_CreditFacilityDomain" xlink:to="us-gaap_CreditFacilityDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CreditFacilityDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Credit Facility [Domain]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DomesticLineOfCreditMember" xlink:type="locator" xlink:label="us-gaap_DomesticLineOfCreditMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DomesticLineOfCreditMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Domestic Line of Credit [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DomesticLineOfCreditMember" xlink:to="us-gaap_DomesticLineOfCreditMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DomesticLineOfCreditMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Domestic Line Of Credit [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LineOfCreditFacilityLineItems" xlink:type="locator" xlink:label="us-gaap_LineOfCreditFacilityLineItems" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LineOfCreditFacilityLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Line of Credit Facility [Line Items]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LineOfCreditFacilityLineItems" xlink:to="us-gaap_LineOfCreditFacilityLineItems_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LineOfCreditFacilityLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Line Of Credit Facility [Line Items]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_CarryingValueOfResidentialMortgageLoansPledgedUnderBlanketCollateralAgreement_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Carrying value of residential mortgage loans pledged under a blanket collateral agreement</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_CarryingValueOfResidentialMortgageLoansPledgedUnderBlanketCollateralAgreement_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Carrying Value Of Residential Mortgage Loans Pledged Under Blanket Collateral Agreement</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_CarryingValueOfFhlbStockPledgedUnderBlanketCollateralAgreement_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Carrying value of FHLB stock pledged under a blanket collateral agreement</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_CarryingValueOfFhlbStockPledgedUnderBlanketCollateralAgreement_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Carrying Value Of Fhlb Stock Pledged Under Blanket Collateral Agreement</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LineOfCreditFacilityMaximumBorrowingCapacity" xlink:type="locator" xlink:label="us-gaap_LineOfCreditFacilityMaximumBorrowingCapacity" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LineOfCreditFacilityMaximumBorrowingCapacity_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Maximum borrowing capacity</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LineOfCreditFacilityMaximumBorrowingCapacity" xlink:to="us-gaap_LineOfCreditFacilityMaximumBorrowingCapacity_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LineOfCreditFacilityMaximumBorrowingCapacity_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Line Of Credit Facility Maximum Borrowing Capacity</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfCorrespondingBanksWithLineOfCreditAvailable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Number of corresponding banks with a line of credit available</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfCorrespondingBanksWithLineOfCreditAvailable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Number Of Corresponding Banks With Line Of Credit Available</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_LineOfCreditFacilityMaximumBorrowingCapacityUnsecuredBasis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Maximum borrowing capacity, unsecured basis</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_LineOfCreditFacilityMaximumBorrowingCapacityUnsecuredBasis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Line Of Credit Facility Maximum Borrowing Capacity Unsecured Basis</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_LineOfCreditFacilityMaximumBorrowingCapacityRequiringCollateral_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Maximum borrowing capacity, requiring collateral</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_LineOfCreditFacilityMaximumBorrowingCapacityRequiringCollateral_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Line Of Credit Facility Maximum Borrowing Capacity Requiring Collateral</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_DebtInstrumentCallProvisionOnTrustSecurities" xlink:type="locator" xlink:label="pbhc_DebtInstrumentCallProvisionOnTrustSecurities" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DebtInstrumentCallProvisionOnTrustSecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The call provision period on the capital securities.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_DebtInstrumentCallProvisionOnTrustSecurities" xlink:to="pbhc_DebtInstrumentCallProvisionOnTrustSecurities_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_DebtInstrumentTermOfVariableRate" xlink:type="locator" xlink:label="pbhc_DebtInstrumentTermOfVariableRate" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DebtInstrumentTermOfVariableRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Term of the interest rate that fluctuates over time as a result of an underling benchmark interest rate or index.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_DebtInstrumentTermOfVariableRate" xlink:to="pbhc_DebtInstrumentTermOfVariableRate_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_JuniorSubordinatedDebtMember" xlink:type="locator" xlink:label="us-gaap_JuniorSubordinatedDebtMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_JuniorSubordinatedDebtMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Junior Subordinated Debentures [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_JuniorSubordinatedDebtMember" xlink:to="us-gaap_JuniorSubordinatedDebtMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_JuniorSubordinatedDebtMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Junior Subordinated Debt [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_VariableRateAxis" xlink:type="locator" xlink:label="us-gaap_VariableRateAxis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_VariableRateAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Variable Rate</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_VariableRateAxis" xlink:to="us-gaap_VariableRateAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_VariableRateAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Variable Rate [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_VariableRateDomain" xlink:type="locator" xlink:label="us-gaap_VariableRateDomain" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_VariableRateDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Variable Rate</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_VariableRateDomain" xlink:to="us-gaap_VariableRateDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_VariableRateDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Variable Rate [Domain]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LondonInterbankOfferedRateLIBORMember" xlink:type="locator" xlink:label="us-gaap_LondonInterbankOfferedRateLIBORMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LondonInterbankOfferedRateLIBORMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">3-Month LIBOR [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LondonInterbankOfferedRateLIBORMember" xlink:to="us-gaap_LondonInterbankOfferedRateLIBORMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LondonInterbankOfferedRateLIBORMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">London Interbank Offered Rate L I B O R [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_PathfinderStatutoryTrustIiMember" xlink:type="locator" xlink:label="pbhc_PathfinderStatutoryTrustIiMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_PathfinderStatutoryTrustIiMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The non-consolidated wholly owned subsidiary trust.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_PathfinderStatutoryTrustIiMember" xlink:to="pbhc_PathfinderStatutoryTrustIiMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_PathfinderStatutoryTrustIiMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Pathfinder Statutory Trust II [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_PathfinderStatutoryTrustIiMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Pathfinder Statutory Trust Ii [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_EquityMethodInvestmentOwnershipPercentage" xlink:type="locator" xlink:label="us-gaap_EquityMethodInvestmentOwnershipPercentage" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EquityMethodInvestmentOwnershipPercentage_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Ownership interest</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_EquityMethodInvestmentOwnershipPercentage" xlink:to="us-gaap_EquityMethodInvestmentOwnershipPercentage_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EquityMethodInvestmentOwnershipPercentage_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Equity Method Investment Ownership Percentage</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DebtInstrumentTerm" xlink:type="locator" xlink:label="us-gaap_DebtInstrumentTerm" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtInstrumentTerm_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Term of debt</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DebtInstrumentTerm" xlink:to="us-gaap_DebtInstrumentTerm_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtInstrumentTerm_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Debt Instrument Term</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DebtInstrumentMaturityDate" xlink:type="locator" xlink:label="us-gaap_DebtInstrumentMaturityDate" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtInstrumentMaturityDate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Maturity date</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DebtInstrumentMaturityDate" xlink:to="us-gaap_DebtInstrumentMaturityDate_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtInstrumentMaturityDate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Debt Instrument Maturity Date</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DebtInstrumentDescriptionOfVariableRateBasis" xlink:type="locator" xlink:label="us-gaap_DebtInstrumentDescriptionOfVariableRateBasis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtInstrumentDescriptionOfVariableRateBasis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Variable rate basis</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DebtInstrumentDescriptionOfVariableRateBasis" xlink:to="us-gaap_DebtInstrumentDescriptionOfVariableRateBasis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtInstrumentDescriptionOfVariableRateBasis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Debt Instrument Description Of Variable Rate Basis</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DebtInstrumentTermOfVariableRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Debt instrument, term of variable rate</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DebtInstrumentTermOfVariableRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Debt Instrument Term Of Variable Rate</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DebtInstrumentBasisSpreadOnVariableRate1" xlink:type="locator" xlink:label="us-gaap_DebtInstrumentBasisSpreadOnVariableRate1" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtInstrumentBasisSpreadOnVariableRate1_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Basis spread on variable rate</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DebtInstrumentBasisSpreadOnVariableRate1" xlink:to="us-gaap_DebtInstrumentBasisSpreadOnVariableRate1_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtInstrumentBasisSpreadOnVariableRate1_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Debt Instrument Basis Spread On Variable Rate1</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DebtInstrumentCallProvisionOnTrustSecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Call provision on trust securities</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DebtInstrumentCallProvisionOnTrustSecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Debt Instrument Call Provision On Trust Securities</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_OriginationAndLegalFees" xlink:type="locator" xlink:label="pbhc_OriginationAndLegalFees" />
    <link:label xml:lang="en-US" xlink:label="pbhc_OriginationAndLegalFees_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The amount of expense provided in the period for origination and legal costs incurred during the period.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_OriginationAndLegalFees" xlink:to="pbhc_OriginationAndLegalFees_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DebtInstrumentInterestRateEffectivePercentage" xlink:type="locator" xlink:label="us-gaap_DebtInstrumentInterestRateEffectivePercentage" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtInstrumentInterestRateEffectivePercentage_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Effective interest rate</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DebtInstrumentInterestRateEffectivePercentage" xlink:to="us-gaap_DebtInstrumentInterestRateEffectivePercentage_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtInstrumentInterestRateEffectivePercentage_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Debt Instrument Interest Rate Effective Percentage</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DebtInstrumentInterestRateStatedPercentage" xlink:type="locator" xlink:label="us-gaap_DebtInstrumentInterestRateStatedPercentage" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtInstrumentInterestRateStatedPercentage_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Subordinated loan interest rate</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DebtInstrumentInterestRateStatedPercentage" xlink:to="us-gaap_DebtInstrumentInterestRateStatedPercentage_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtInstrumentInterestRateStatedPercentage_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Debt Instrument Interest Rate Stated Percentage</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_OriginationAndLegalFees_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Origination and legal fees</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_OriginationAndLegalFees_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Origination And Legal Fees</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ProceedsFromIssuanceOfSubordinatedLongTermDebt" xlink:type="locator" xlink:label="us-gaap_ProceedsFromIssuanceOfSubordinatedLongTermDebt" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ProceedsFromIssuanceOfSubordinatedLongTermDebt_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Proceeds from issuance of subordinate notes</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ProceedsFromIssuanceOfSubordinatedLongTermDebt" xlink:to="us-gaap_ProceedsFromIssuanceOfSubordinatedLongTermDebt_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ProceedsFromIssuanceOfSubordinatedLongTermDebt_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Proceeds From Issuance Of Subordinated Long Term Debt</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_SubordinatedBorrowingInterestRate" xlink:type="locator" xlink:label="us-gaap_SubordinatedBorrowingInterestRate" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SubordinatedBorrowingInterestRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Subordinated borrowing, interest rate</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_SubordinatedBorrowingInterestRate" xlink:to="us-gaap_SubordinatedBorrowingInterestRate_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SubordinatedBorrowingInterestRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Subordinated Borrowing Interest Rate</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_SubordinatedBorrowingDueDate" xlink:type="locator" xlink:label="us-gaap_SubordinatedBorrowingDueDate" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SubordinatedBorrowingDueDate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Subordinated note, maturity date</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_SubordinatedBorrowingDueDate" xlink:to="us-gaap_SubordinatedBorrowingDueDate_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SubordinatedBorrowingDueDate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Subordinated Borrowing Due Date</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DerivativeBasisSpreadOnVariableRate" xlink:type="locator" xlink:label="us-gaap_DerivativeBasisSpreadOnVariableRate" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DerivativeBasisSpreadOnVariableRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Spread on variable rate</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DerivativeBasisSpreadOnVariableRate" xlink:to="us-gaap_DerivativeBasisSpreadOnVariableRate_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DerivativeBasisSpreadOnVariableRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Derivative Basis Spread On Variable Rate</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_SubordinatedTermLoanMember" xlink:type="locator" xlink:label="pbhc_SubordinatedTermLoanMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_SubordinatedTermLoanMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">This element represents domestic or foreign subordinated debt with maturity term.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_SubordinatedTermLoanMember" xlink:to="pbhc_SubordinatedTermLoanMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_SubordinatedTermLoanMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Subordinated Loan [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_SubordinatedTermLoanMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Subordinated Term Loan [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_SubordinatedLongTermDebt" xlink:type="locator" xlink:label="us-gaap_SubordinatedLongTermDebt" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SubordinatedLongTermDebt_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Total subordinated loans</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_SubordinatedLongTermDebt" xlink:to="us-gaap_SubordinatedLongTermDebt_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SubordinatedLongTermDebt_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Subordinated Long Term Debt</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_MaturitiesOfLongTermDebtInterestRateAbstract" xlink:type="locator" xlink:label="pbhc_MaturitiesOfLongTermDebtInterestRateAbstract" />
    <link:label xml:lang="en-US" xlink:label="pbhc_MaturitiesOfLongTermDebtInterestRateAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Maturities Of Long Term Debt Interest Rate Abstract.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_MaturitiesOfLongTermDebtInterestRateAbstract" xlink:to="pbhc_MaturitiesOfLongTermDebtInterestRateAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_MaturitiesOfLongTermDebtInterestRateAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Maturities Of Long Term Debt Interest Rate [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_LongTermDebtMaturitiesSummaryDueInWithinFiveYears_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Due within 5 years</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_LongTermDebtMaturitiesSummaryDueInWithinFiveYears_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Long Term Debt Maturities Summary Due In Within Five Years</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_LongTermDebtMaturitiesSummaryDueInWithinTenYears_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Due within 10 years</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_LongTermDebtMaturitiesSummaryDueInWithinTenYears_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Long Term Debt Maturities Summary Due In Within Ten Years</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_LongTermDebtMaturitiesSummaryDueInWithinSeventeenYears_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Due within 17 years</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_LongTermDebtMaturitiesSummaryDueInWithinSeventeenYears_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Long Term Debt Maturities Summary Due In Within Seventeen Years</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_SubordinatedLongTermDebt_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Total subordinated loans</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_LongTermDebtMaturitiesSummaryByInterestRateWithinFiveYearsInterestRate" xlink:type="locator" xlink:label="pbhc_LongTermDebtMaturitiesSummaryByInterestRateWithinFiveYearsInterestRate" />
    <link:label xml:lang="en-US" xlink:label="pbhc_LongTermDebtMaturitiesSummaryByInterestRateWithinFiveYearsInterestRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Represents the interest rate applicable to long term debt due within five years of the Balance Sheet date.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_LongTermDebtMaturitiesSummaryByInterestRateWithinFiveYearsInterestRate" xlink:to="pbhc_LongTermDebtMaturitiesSummaryByInterestRateWithinFiveYearsInterestRate_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_LongTermDebtMaturitiesSummaryByInterestRateWithinTenYearsInterestRate" xlink:type="locator" xlink:label="pbhc_LongTermDebtMaturitiesSummaryByInterestRateWithinTenYearsInterestRate" />
    <link:label xml:lang="en-US" xlink:label="pbhc_LongTermDebtMaturitiesSummaryByInterestRateWithinTenYearsInterestRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Represents the interest rate applicable to long term debt due within ten years of the Balance Sheet date.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_LongTermDebtMaturitiesSummaryByInterestRateWithinTenYearsInterestRate" xlink:to="pbhc_LongTermDebtMaturitiesSummaryByInterestRateWithinTenYearsInterestRate_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_LongTermDebtMaturitiesSummaryByInterestRateWithinFiveYearsInterestRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Due within 5 years</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_LongTermDebtMaturitiesSummaryByInterestRateWithinFiveYearsInterestRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Long Term Debt Maturities Summary By Interest Rate Within Five Years Interest Rate</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_LongTermDebtMaturitiesSummaryByInterestRateWithinTenYearsInterestRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Due within 10 years</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_LongTermDebtMaturitiesSummaryByInterestRateWithinTenYearsInterestRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Long Term Debt Maturities Summary By Interest Rate Within Ten Years Interest Rate</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtInstrumentBasisSpreadOnVariableRate1_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Due within 17 years</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalAfterYearFive" xlink:type="locator" xlink:label="us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalAfterYearFive" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalAfterYearFive_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Thereafter</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalAfterYearFive" xlink:to="us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalAfterYearFive_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalAfterYearFive_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Long Term Debt Maturities Repayments Of Principal After Year Five</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_MinimumYearsOfServiceToParticipateInHealthAndLifeInsuranceBenefitsAsOfJanuary11995" xlink:type="locator" xlink:label="pbhc_MinimumYearsOfServiceToParticipateInHealthAndLifeInsuranceBenefitsAsOfJanuary11995" />
    <link:label xml:lang="en-US" xlink:label="pbhc_MinimumYearsOfServiceToParticipateInHealthAndLifeInsuranceBenefitsAsOfJanuary11995_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The minimum years of service before an employee can be eligible to participate in the health and life insurance benefits as of January 1, 1995.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_MinimumYearsOfServiceToParticipateInHealthAndLifeInsuranceBenefitsAsOfJanuary11995" xlink:to="pbhc_MinimumYearsOfServiceToParticipateInHealthAndLifeInsuranceBenefitsAsOfJanuary11995_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlansAndOtherPostretirementBenefitPlansDisclosuresTable" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlansAndOtherPostretirementBenefitPlansDisclosuresTable" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlansAndOtherPostretirementBenefitPlansDisclosuresTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Defined Benefit Plans And Other Postretirement Benefit Plans Disclosures [Table]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlansAndOtherPostretirementBenefitPlansDisclosuresTable" xlink:to="us-gaap_DefinedBenefitPlansAndOtherPostretirementBenefitPlansDisclosuresTable_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlansAndOtherPostretirementBenefitPlansDisclosuresTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plans And Other Postretirement Benefit Plans Disclosures [Table]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_RetirementPlanTypeAxis" xlink:type="locator" xlink:label="us-gaap_RetirementPlanTypeAxis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RetirementPlanTypeAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Retirement Plan Type</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_RetirementPlanTypeAxis" xlink:to="us-gaap_RetirementPlanTypeAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RetirementPlanTypeAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Retirement Plan Type [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_RetirementPlanTypeDomain" xlink:type="locator" xlink:label="us-gaap_RetirementPlanTypeDomain" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RetirementPlanTypeDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Retirement Plan Type</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_RetirementPlanTypeDomain" xlink:to="us-gaap_RetirementPlanTypeDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RetirementPlanTypeDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Retirement Plan Type [Domain]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_PensionPlansDefinedBenefitMember" xlink:type="locator" xlink:label="us-gaap_PensionPlansDefinedBenefitMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PensionPlansDefinedBenefitMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Pension Benefits [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_PensionPlansDefinedBenefitMember" xlink:to="us-gaap_PensionPlansDefinedBenefitMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PensionPlansDefinedBenefitMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Pension Plans Defined Benefit [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_OtherPostretirementBenefitPlansDefinedBenefitMember" xlink:type="locator" xlink:label="us-gaap_OtherPostretirementBenefitPlansDefinedBenefitMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherPostretirementBenefitPlansDefinedBenefitMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Postretirement Benefits [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_OtherPostretirementBenefitPlansDefinedBenefitMember" xlink:to="us-gaap_OtherPostretirementBenefitPlansDefinedBenefitMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherPostretirementBenefitPlansDefinedBenefitMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Other Postretirement Benefit Plans Defined Benefit [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPostretirementHealthCoverageMember" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPostretirementHealthCoverageMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPostretirementHealthCoverageMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Postretirement Health Care Plan [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPostretirementHealthCoverageMember" xlink:to="us-gaap_DefinedBenefitPostretirementHealthCoverageMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPostretirementHealthCoverageMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Postretirement Health Coverage [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanByPlanAssetCategoriesAxis" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanByPlanAssetCategoriesAxis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanByPlanAssetCategoriesAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Defined Benefit Plan, Plan Assets, Category</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanByPlanAssetCategoriesAxis" xlink:to="us-gaap_DefinedBenefitPlanByPlanAssetCategoriesAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanByPlanAssetCategoriesAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan By Plan Asset Categories [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_PlanAssetCategoriesDomain" xlink:type="locator" xlink:label="us-gaap_PlanAssetCategoriesDomain" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PlanAssetCategoriesDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Defined Benefit Plan, Plan Assets, Category</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_PlanAssetCategoriesDomain" xlink:to="us-gaap_PlanAssetCategoriesDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PlanAssetCategoriesDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Plan Asset Categories [Domain]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_EquitySecuritiesMember" xlink:type="locator" xlink:label="us-gaap_EquitySecuritiesMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EquitySecuritiesMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Equity Securities [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_EquitySecuritiesMember" xlink:to="us-gaap_EquitySecuritiesMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EquitySecuritiesMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Equity Securities [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FixedIncomeFundsMember" xlink:type="locator" xlink:label="us-gaap_FixedIncomeFundsMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FixedIncomeFundsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Fixed Income Securities [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FixedIncomeFundsMember" xlink:to="us-gaap_FixedIncomeFundsMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FixedIncomeFundsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Fixed Income Funds [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MutualFundsFixedIncomeIntermediateDurationMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Mutual Funds - Fixed Income Intermediate duration [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MutualFundsFixedIncomeIntermediateDurationMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Mutual Funds Fixed Income Intermediate Duration [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MutualFundsFixedIncomeLongDurationMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Mutual Funds-Fixed Income Long Duration [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MutualFundsFixedIncomeLongDurationMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Mutual Funds Fixed Income Long Duration [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_CashEquivalentsMember" xlink:type="locator" xlink:label="us-gaap_CashEquivalentsMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CashEquivalentsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Cash Equivalents [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_CashEquivalentsMember" xlink:to="us-gaap_CashEquivalentsMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CashEquivalentsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Cash Equivalents [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/srt/2020/elts/srt-2020-01-31.xsd#srt_StatementScenarioAxis" xlink:type="locator" xlink:label="srt_StatementScenarioAxis" />
    <link:label xml:lang="en-US" xlink:label="srt_StatementScenarioAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Scenario</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="srt_StatementScenarioAxis" xlink:to="srt_StatementScenarioAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="srt_StatementScenarioAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Statement Scenario [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/srt/2020/elts/srt-2020-01-31.xsd#srt_ScenarioUnspecifiedDomain" xlink:type="locator" xlink:label="srt_ScenarioUnspecifiedDomain" />
    <link:label xml:lang="en-US" xlink:label="srt_ScenarioUnspecifiedDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Scenario</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="srt_ScenarioUnspecifiedDomain" xlink:to="srt_ScenarioUnspecifiedDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="srt_ScenarioUnspecifiedDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Scenario Unspecified [Domain]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/srt/2020/elts/srt-2020-01-31.xsd#srt_ScenarioForecastMember" xlink:type="locator" xlink:label="srt_ScenarioForecastMember" />
    <link:label xml:lang="en-US" xlink:label="srt_ScenarioForecastMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Scenario Forecast [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="srt_ScenarioForecastMember" xlink:to="srt_ScenarioForecastMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="srt_ScenarioForecastMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Scenario Forecast [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_DirectorsAndCertainExecutiveOfficersMember" xlink:type="locator" xlink:label="pbhc_DirectorsAndCertainExecutiveOfficersMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DirectorsAndCertainExecutiveOfficersMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Officers of the entity that are appointed by the board of directors.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_DirectorsAndCertainExecutiveOfficersMember" xlink:to="pbhc_DirectorsAndCertainExecutiveOfficersMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DirectorsAndCertainExecutiveOfficersMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Directors and Certain Executive Officers [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DirectorsAndCertainExecutiveOfficersMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Directors And Certain Executive Officers [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_SupplementalEmployeeRetirementPlanDefinedBenefitMember" xlink:type="locator" xlink:label="us-gaap_SupplementalEmployeeRetirementPlanDefinedBenefitMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SupplementalEmployeeRetirementPlanDefinedBenefitMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Supplemental Executive Retirement Plans [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_SupplementalEmployeeRetirementPlanDefinedBenefitMember" xlink:to="us-gaap_SupplementalEmployeeRetirementPlanDefinedBenefitMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SupplementalEmployeeRetirementPlanDefinedBenefitMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Supplemental Employee Retirement Plan Defined Benefit [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlansAndOtherPostretirementBenefitPlansTableTextBlockLineItems" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlansAndOtherPostretirementBenefitPlansTableTextBlockLineItems" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlansAndOtherPostretirementBenefitPlansTableTextBlockLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Defined Benefit Plans And Other Postretirement Benefit Plans Table Text Block [Line Items]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlansAndOtherPostretirementBenefitPlansTableTextBlockLineItems" xlink:to="us-gaap_DefinedBenefitPlansAndOtherPostretirementBenefitPlansTableTextBlockLineItems_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlansAndOtherPostretirementBenefitPlansTableTextBlockLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plans And Other Postretirement Benefit Plans Table Text Block [Line Items]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MinimumYearsOfServiceToParticipateInHealthAndLifeInsuranceBenefitsAsOfJanuary11995_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Minimum years of service to participate in the health and life insurance benefits as of January 1, 1995</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MinimumYearsOfServiceToParticipateInHealthAndLifeInsuranceBenefitsAsOfJanuary11995_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Minimum Years Of Service To Participate In Health And Life Insurance Benefits As Of January11995</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanChangeInBenefitObligationRollForward" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanChangeInBenefitObligationRollForward" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanChangeInBenefitObligationRollForward_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Change in Benefit Obligation [Roll Forward]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanChangeInBenefitObligationRollForward" xlink:to="us-gaap_DefinedBenefitPlanChangeInBenefitObligationRollForward_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanChangeInBenefitObligationRollForward_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Change In Benefit Obligation Roll Forward</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanBenefitObligation" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanBenefitObligation" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanBenefitObligation_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/periodStartLabel">Benefit obligations at beginning of year</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanBenefitObligation" xlink:to="us-gaap_DefinedBenefitPlanBenefitObligation_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanBenefitObligation_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Benefit Obligation</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanServiceCost" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanServiceCost" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanServiceCost_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Service cost</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanServiceCost" xlink:to="us-gaap_DefinedBenefitPlanServiceCost_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanServiceCost_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Service Cost</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanInterestCost" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanInterestCost" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanInterestCost_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Interest cost</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanInterestCost" xlink:to="us-gaap_DefinedBenefitPlanInterestCost_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanInterestCost_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Interest Cost</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanBenefitObligationContributionsByPlanParticipant" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanBenefitObligationContributionsByPlanParticipant" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanBenefitObligationContributionsByPlanParticipant_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Plan participants&apos; contribution</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanBenefitObligationContributionsByPlanParticipant" xlink:to="us-gaap_DefinedBenefitPlanBenefitObligationContributionsByPlanParticipant_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanBenefitObligationContributionsByPlanParticipant_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Benefit Obligation Contributions By Plan Participant</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanActuarialGainLoss" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanActuarialGainLoss" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanActuarialGainLoss_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Actuarial loss (gain)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanActuarialGainLoss" xlink:to="us-gaap_DefinedBenefitPlanActuarialGainLoss_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanActuarialGainLoss_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Actuarial Gain Loss</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanBenefitObligationBenefitsPaid" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanBenefitObligationBenefitsPaid" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanBenefitObligationBenefitsPaid_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Benefits paid</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanBenefitObligationBenefitsPaid" xlink:to="us-gaap_DefinedBenefitPlanBenefitObligationBenefitsPaid_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanBenefitObligationBenefitsPaid_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Benefit Obligation Benefits Paid</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanBenefitObligation_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/periodEndLabel">Benefit obligations at end of year</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanChangeInFairValueOfPlanAssetsRollForward" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanChangeInFairValueOfPlanAssetsRollForward" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanChangeInFairValueOfPlanAssetsRollForward_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Change in Plan Assets [Roll Forward]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanChangeInFairValueOfPlanAssetsRollForward" xlink:to="us-gaap_DefinedBenefitPlanChangeInFairValueOfPlanAssetsRollForward_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanChangeInFairValueOfPlanAssetsRollForward_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Change In Fair Value Of Plan Assets Roll Forward</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanFairValueOfPlanAssets" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanFairValueOfPlanAssets" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanFairValueOfPlanAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/periodStartLabel">Fair value of plan assets at beginning of year</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanFairValueOfPlanAssets" xlink:to="us-gaap_DefinedBenefitPlanFairValueOfPlanAssets_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanFairValueOfPlanAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Fair Value Of Plan Assets</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanActualReturnOnPlanAssets" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanActualReturnOnPlanAssets" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanActualReturnOnPlanAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Actual return on plan assets</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanActualReturnOnPlanAssets" xlink:to="us-gaap_DefinedBenefitPlanActualReturnOnPlanAssets_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanActualReturnOnPlanAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Actual Return On Plan Assets</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanPlanAssetsBenefitsPaid" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanPlanAssetsBenefitsPaid" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanPlanAssetsBenefitsPaid_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Benefits paid</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanPlanAssetsBenefitsPaid" xlink:to="us-gaap_DefinedBenefitPlanPlanAssetsBenefitsPaid_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanPlanAssetsBenefitsPaid_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Plan Assets Benefits Paid</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanPlanAssetsContributionsByPlanParticipant" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanPlanAssetsContributionsByPlanParticipant" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanPlanAssetsContributionsByPlanParticipant_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Plan participants&apos; contribution</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanPlanAssetsContributionsByPlanParticipant" xlink:to="us-gaap_DefinedBenefitPlanPlanAssetsContributionsByPlanParticipant_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanPlanAssetsContributionsByPlanParticipant_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Plan Assets Contributions By Plan Participant</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanContributionsByEmployer" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanContributionsByEmployer" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanContributionsByEmployer_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Employer contributions</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanContributionsByEmployer" xlink:to="us-gaap_DefinedBenefitPlanContributionsByEmployer_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanContributionsByEmployer_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Contributions By Employer</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanFairValueOfPlanAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/periodEndLabel">Fair value of plan assets at end of year</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanFundedStatusOfPlan" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanFundedStatusOfPlan" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanFundedStatusOfPlan_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Funded (unfunded) status - asset (liability)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanFundedStatusOfPlan" xlink:to="us-gaap_DefinedBenefitPlanFundedStatusOfPlan_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanFundedStatusOfPlan_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Funded Status Of Plan</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_DefinedBenefitPlanAmountsRecognizedInAccumulatedOtherComprehensiveLossAbstract" xlink:type="locator" xlink:label="pbhc_DefinedBenefitPlanAmountsRecognizedInAccumulatedOtherComprehensiveLossAbstract" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DefinedBenefitPlanAmountsRecognizedInAccumulatedOtherComprehensiveLossAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Defined benefit plan amounts recognized in accumulated other comprehensive loss.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_DefinedBenefitPlanAmountsRecognizedInAccumulatedOtherComprehensiveLossAbstract" xlink:to="pbhc_DefinedBenefitPlanAmountsRecognizedInAccumulatedOtherComprehensiveLossAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DefinedBenefitPlanAmountsRecognizedInAccumulatedOtherComprehensiveLossAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Amounts Recognized in Accumulated Other Comprehensive Loss [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DefinedBenefitPlanAmountsRecognizedInAccumulatedOtherComprehensiveLossAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Amounts Recognized In Accumulated Other Comprehensive Loss [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanAccumulatedOtherComprehensiveIncomeBeforeTax" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanAccumulatedOtherComprehensiveIncomeBeforeTax" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanAccumulatedOtherComprehensiveIncomeBeforeTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Net loss</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanAccumulatedOtherComprehensiveIncomeBeforeTax" xlink:to="us-gaap_DefinedBenefitPlanAccumulatedOtherComprehensiveIncomeBeforeTax_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanAccumulatedOtherComprehensiveIncomeBeforeTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Accumulated Other Comprehensive Income Before Tax</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DefinedBenefitPlanAccumulatedOtherComprehensiveIncomeLossTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Tax Effect</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DefinedBenefitPlanAccumulatedOtherComprehensiveIncomeLossTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Accumulated Other Comprehensive Income Loss Tax</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_AccumulatedOtherComprehensiveIncomeLossDefinedBenefitPensionAndOtherPostretirementPlansNetOfTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Accumulated other comprehensive income after tax</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_GainsOrLossesGreaterOfBenefitObligationOrFairValueOfAssetsAmortizedOverAverageRemainingServicePeriod" xlink:type="locator" xlink:label="pbhc_GainsOrLossesGreaterOfBenefitObligationOrFairValueOfAssetsAmortizedOverAverageRemainingServicePeriod" />
    <link:label xml:lang="en-US" xlink:label="pbhc_GainsOrLossesGreaterOfBenefitObligationOrFairValueOfAssetsAmortizedOverAverageRemainingServicePeriod_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The percentage of gains or losses greater of the benefit obligation or the fair value of assets amortized over the average remaining service period.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_GainsOrLossesGreaterOfBenefitObligationOrFairValueOfAssetsAmortizedOverAverageRemainingServicePeriod" xlink:to="pbhc_GainsOrLossesGreaterOfBenefitObligationOrFairValueOfAssetsAmortizedOverAverageRemainingServicePeriod_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_GainsOrLossesGreaterOfBenefitObligationOrFairValueOfAssetsAmortizedOverAverageRemainingServicePeriod_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Gains or losses greater of the benefit obligation or the fair value of assets amortized over the average remaining service period</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_GainsOrLossesGreaterOfBenefitObligationOrFairValueOfAssetsAmortizedOverAverageRemainingServicePeriod_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Gains Or Losses Greater Of Benefit Obligation Or Fair Value Of Assets Amortized Over Average Remaining Service Period</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanWeightedAverageAssumptionsUsedInCalculatingBenefitObligationAbstract" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanWeightedAverageAssumptionsUsedInCalculatingBenefitObligationAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanWeightedAverageAssumptionsUsedInCalculatingBenefitObligationAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Significant Assumptions Used in Determining Benefit Obligations [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanWeightedAverageAssumptionsUsedInCalculatingBenefitObligationAbstract" xlink:to="us-gaap_DefinedBenefitPlanWeightedAverageAssumptionsUsedInCalculatingBenefitObligationAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanWeightedAverageAssumptionsUsedInCalculatingBenefitObligationAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Weighted Average Assumptions Used In Calculating Benefit Obligation [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanAssumptionsUsedCalculatingBenefitObligationDiscountRate" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanAssumptionsUsedCalculatingBenefitObligationDiscountRate" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanAssumptionsUsedCalculatingBenefitObligationDiscountRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Weighted average discount rate</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanAssumptionsUsedCalculatingBenefitObligationDiscountRate" xlink:to="us-gaap_DefinedBenefitPlanAssumptionsUsedCalculatingBenefitObligationDiscountRate_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanAssumptionsUsedCalculatingBenefitObligationDiscountRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Assumptions Used Calculating Benefit Obligation Discount Rate</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanBenefitObligation_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Accumulated benefit obligation</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanAssumptionsUsedCalculatingNetPeriodicBenefitCostRateOfCompensationIncrease" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanAssumptionsUsedCalculatingNetPeriodicBenefitCostRateOfCompensationIncrease" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanAssumptionsUsedCalculatingNetPeriodicBenefitCostRateOfCompensationIncrease_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Rate of increase in future compensation levels</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanAssumptionsUsedCalculatingNetPeriodicBenefitCostRateOfCompensationIncrease" xlink:to="us-gaap_DefinedBenefitPlanAssumptionsUsedCalculatingNetPeriodicBenefitCostRateOfCompensationIncrease_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanAssumptionsUsedCalculatingNetPeriodicBenefitCostRateOfCompensationIncrease_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Assumptions Used Calculating Net Periodic Benefit Cost Rate Of Compensation Increase</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanAssumedHealthCareCostTrendRatesAbstract" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanAssumedHealthCareCostTrendRatesAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanAssumedHealthCareCostTrendRatesAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Assumed Health Care Cost Trend Rates [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanAssumedHealthCareCostTrendRatesAbstract" xlink:to="us-gaap_DefinedBenefitPlanAssumedHealthCareCostTrendRatesAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanAssumedHealthCareCostTrendRatesAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Assumed Health Care Cost Trend Rates [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanHealthCareCostTrendRateAssumedNextFiscalYear" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanHealthCareCostTrendRateAssumedNextFiscalYear" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanHealthCareCostTrendRateAssumedNextFiscalYear_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Health care cost trend rate assumption</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanHealthCareCostTrendRateAssumedNextFiscalYear" xlink:to="us-gaap_DefinedBenefitPlanHealthCareCostTrendRateAssumedNextFiscalYear_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanHealthCareCostTrendRateAssumedNextFiscalYear_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Health Care Cost Trend Rate Assumed Next Fiscal Year</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanUltimateHealthCareCostTrendRate1" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanUltimateHealthCareCostTrendRate1" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanUltimateHealthCareCostTrendRate1_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Ultimate health care cost trend rate</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanUltimateHealthCareCostTrendRate1" xlink:to="us-gaap_DefinedBenefitPlanUltimateHealthCareCostTrendRate1_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanUltimateHealthCareCostTrendRate1_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Ultimate Health Care Cost Trend Rate1</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanYearHealthCareCostTrendRateReachesUltimateTrendRate" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanYearHealthCareCostTrendRateReachesUltimateTrendRate" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanYearHealthCareCostTrendRateReachesUltimateTrendRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Year that rate reaches ultimate trend rate</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanYearHealthCareCostTrendRateReachesUltimateTrendRate" xlink:to="us-gaap_DefinedBenefitPlanYearHealthCareCostTrendRateReachesUltimateTrendRate_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanYearHealthCareCostTrendRateReachesUltimateTrendRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Year Health Care Cost Trend Rate Reaches Ultimate Trend Rate</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanNetPeriodicBenefitCostAbstract" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanNetPeriodicBenefitCostAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanNetPeriodicBenefitCostAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Composition of Net Periodic Benefit Plan Cost [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanNetPeriodicBenefitCostAbstract" xlink:to="us-gaap_DefinedBenefitPlanNetPeriodicBenefitCostAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanNetPeriodicBenefitCostAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Net Periodic Benefit Cost [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanExpectedReturnOnPlanAssets" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanExpectedReturnOnPlanAssets" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanExpectedReturnOnPlanAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Expected return on plan assets</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanExpectedReturnOnPlanAssets" xlink:to="us-gaap_DefinedBenefitPlanExpectedReturnOnPlanAssets_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanExpectedReturnOnPlanAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Expected Return On Plan Assets</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanAmortizationOfTransitionAssetObligation" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanAmortizationOfTransitionAssetObligation" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanAmortizationOfTransitionAssetObligation_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Amortization of transition obligation</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanAmortizationOfTransitionAssetObligation" xlink:to="us-gaap_DefinedBenefitPlanAmortizationOfTransitionAssetObligation_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanAmortizationOfTransitionAssetObligation_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Amortization Of Transition Asset Obligation</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanAmortizationOfGainsLosses" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanAmortizationOfGainsLosses" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanAmortizationOfGainsLosses_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Amortization of net losses</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanAmortizationOfGainsLosses" xlink:to="us-gaap_DefinedBenefitPlanAmortizationOfGainsLosses_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanAmortizationOfGainsLosses_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Amortization Of Gains Losses</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanAmortizationOfPriorServiceCostCredit" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanAmortizationOfPriorServiceCostCredit" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanAmortizationOfPriorServiceCostCredit_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Amortization of unrecognized past service liability</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanAmortizationOfPriorServiceCostCredit" xlink:to="us-gaap_DefinedBenefitPlanAmortizationOfPriorServiceCostCredit_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanAmortizationOfPriorServiceCostCredit_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Amortization Of Prior Service Cost Credit</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanNetPeriodicBenefitCost" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanNetPeriodicBenefitCost" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanNetPeriodicBenefitCost_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Net periodic benefit plan (benefit) cost</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanNetPeriodicBenefitCost" xlink:to="us-gaap_DefinedBenefitPlanNetPeriodicBenefitCost_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanNetPeriodicBenefitCost_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Net Periodic Benefit Cost</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanWeightedAverageAssumptionsUsedInCalculatingNetPeriodicBenefitCostAbstract" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanWeightedAverageAssumptionsUsedInCalculatingNetPeriodicBenefitCostAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanWeightedAverageAssumptionsUsedInCalculatingNetPeriodicBenefitCostAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Significant Assumptions Used in Determining Net Periodic Benefit Plan Cost [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanWeightedAverageAssumptionsUsedInCalculatingNetPeriodicBenefitCostAbstract" xlink:to="us-gaap_DefinedBenefitPlanWeightedAverageAssumptionsUsedInCalculatingNetPeriodicBenefitCostAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanWeightedAverageAssumptionsUsedInCalculatingNetPeriodicBenefitCostAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Weighted Average Assumptions Used In Calculating Net Periodic Benefit Cost [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanAssumptionsUsedCalculatingNetPeriodicBenefitCostDiscountRate" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanAssumptionsUsedCalculatingNetPeriodicBenefitCostDiscountRate" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanAssumptionsUsedCalculatingNetPeriodicBenefitCostDiscountRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Weighted average discount rate</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanAssumptionsUsedCalculatingNetPeriodicBenefitCostDiscountRate" xlink:to="us-gaap_DefinedBenefitPlanAssumptionsUsedCalculatingNetPeriodicBenefitCostDiscountRate_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanAssumptionsUsedCalculatingNetPeriodicBenefitCostDiscountRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Assumptions Used Calculating Net Periodic Benefit Cost Discount Rate</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanAssumptionsUsedCalculatingNetPeriodicBenefitCostExpectedLongTermReturnOnAssets" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanAssumptionsUsedCalculatingNetPeriodicBenefitCostExpectedLongTermReturnOnAssets" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanAssumptionsUsedCalculatingNetPeriodicBenefitCostExpectedLongTermReturnOnAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Expected long term rate of return on plan assets</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanAssumptionsUsedCalculatingNetPeriodicBenefitCostExpectedLongTermReturnOnAssets" xlink:to="us-gaap_DefinedBenefitPlanAssumptionsUsedCalculatingNetPeriodicBenefitCostExpectedLongTermReturnOnAssets_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanAssumptionsUsedCalculatingNetPeriodicBenefitCostExpectedLongTermReturnOnAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Assumptions Used Calculating Net Periodic Benefit Cost Expected Long Term Return On Assets</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_DefinedBenefitPlanAssumptionsUsedCalculatingNetPeriodicBenefitCostRealRateLongTermReturnOnAssets" xlink:type="locator" xlink:label="pbhc_DefinedBenefitPlanAssumptionsUsedCalculatingNetPeriodicBenefitCostRealRateLongTermReturnOnAssets" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DefinedBenefitPlanAssumptionsUsedCalculatingNetPeriodicBenefitCostRealRateLongTermReturnOnAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">An assumption as to the real rate of return on plan assets reflecting the average rate of earnings expected on the funds invested or to be invested to provide for the benefits included in the benefit obligation.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_DefinedBenefitPlanAssumptionsUsedCalculatingNetPeriodicBenefitCostRealRateLongTermReturnOnAssets" xlink:to="pbhc_DefinedBenefitPlanAssumptionsUsedCalculatingNetPeriodicBenefitCostRealRateLongTermReturnOnAssets_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_DefinedBenefitPlanAssumptionsUsedCalculatingBenefitObligationLongTermInflationRate" xlink:type="locator" xlink:label="pbhc_DefinedBenefitPlanAssumptionsUsedCalculatingBenefitObligationLongTermInflationRate" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DefinedBenefitPlanAssumptionsUsedCalculatingBenefitObligationLongTermInflationRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The estimated long-term inflation rate during the period.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_DefinedBenefitPlanAssumptionsUsedCalculatingBenefitObligationLongTermInflationRate" xlink:to="pbhc_DefinedBenefitPlanAssumptionsUsedCalculatingBenefitObligationLongTermInflationRate_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_DefinedBenefitPlanAssumptionsUsedCalculatingNetPeriodicBenefitCostExpectedLongTermReturnOnAssetsNextFiscalYear" xlink:type="locator" xlink:label="pbhc_DefinedBenefitPlanAssumptionsUsedCalculatingNetPeriodicBenefitCostExpectedLongTermReturnOnAssetsNextFiscalYear" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DefinedBenefitPlanAssumptionsUsedCalculatingNetPeriodicBenefitCostExpectedLongTermReturnOnAssetsNextFiscalYear_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">An assumption as to the rate of return on plan assets reflecting the average rate of earnings expected on the funds invested or to be invested to provide for the benefits included in the benefit obligation in the next fiscal year.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_DefinedBenefitPlanAssumptionsUsedCalculatingNetPeriodicBenefitCostExpectedLongTermReturnOnAssetsNextFiscalYear" xlink:to="pbhc_DefinedBenefitPlanAssumptionsUsedCalculatingNetPeriodicBenefitCostExpectedLongTermReturnOnAssetsNextFiscalYear_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_DefinedBenefitPlanAssumptionsUsedCalculatingDecreaseInBasisPointExpectedLongTermReturnOnAssets" xlink:type="locator" xlink:label="pbhc_DefinedBenefitPlanAssumptionsUsedCalculatingDecreaseInBasisPointExpectedLongTermReturnOnAssets" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DefinedBenefitPlanAssumptionsUsedCalculatingDecreaseInBasisPointExpectedLongTermReturnOnAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">An decrease assumption as to the rate of return on plan assets reflecting the average rate of earnings expected on the funds invested or to be invested to provide for the benefits included in the benefit obligation.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_DefinedBenefitPlanAssumptionsUsedCalculatingDecreaseInBasisPointExpectedLongTermReturnOnAssets" xlink:to="pbhc_DefinedBenefitPlanAssumptionsUsedCalculatingDecreaseInBasisPointExpectedLongTermReturnOnAssets_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_DefinedBenefitPlanDecreaseInExpectedReturnOnLongTermRateOfReturn" xlink:type="locator" xlink:label="pbhc_DefinedBenefitPlanDecreaseInExpectedReturnOnLongTermRateOfReturn" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DefinedBenefitPlanDecreaseInExpectedReturnOnLongTermRateOfReturn_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">An decrease of amount calculated as a basis for determining the extent of delayed recognition of the effects of changes in the fair value of assets. The expected return is determined based on the expected long-term rate of return on plan assets.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_DefinedBenefitPlanDecreaseInExpectedReturnOnLongTermRateOfReturn" xlink:to="pbhc_DefinedBenefitPlanDecreaseInExpectedReturnOnLongTermRateOfReturn_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_DefinedBenefitPlanPercentageOfDecreaseInExpectedReturnOnLongTermRateOfReturn" xlink:type="locator" xlink:label="pbhc_DefinedBenefitPlanPercentageOfDecreaseInExpectedReturnOnLongTermRateOfReturn" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DefinedBenefitPlanPercentageOfDecreaseInExpectedReturnOnLongTermRateOfReturn_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Defined benefit plan percentage of decrease in expected return on long term rate of return.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_DefinedBenefitPlanPercentageOfDecreaseInExpectedReturnOnLongTermRateOfReturn" xlink:to="pbhc_DefinedBenefitPlanPercentageOfDecreaseInExpectedReturnOnLongTermRateOfReturn_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanFairValueOfPlanAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Fair value of plan assets</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DefinedBenefitPlanAssumptionsUsedCalculatingNetPeriodicBenefitCostRealRateLongTermReturnOnAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Real rate of return on assets assumption</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DefinedBenefitPlanAssumptionsUsedCalculatingNetPeriodicBenefitCostRealRateLongTermReturnOnAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Assumptions Used Calculating Net Periodic Benefit Cost Real Rate Long Term Return On Assets</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DefinedBenefitPlanAssumptionsUsedCalculatingBenefitObligationLongTermInflationRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Long-term inflation rate</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DefinedBenefitPlanAssumptionsUsedCalculatingBenefitObligationLongTermInflationRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Assumptions Used Calculating Benefit Obligation Long Term Inflation Rate</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DefinedBenefitPlanAssumptionsUsedCalculatingNetPeriodicBenefitCostExpectedLongTermReturnOnAssetsNextFiscalYear_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Expected long term rate of return on plan assets - 2021</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DefinedBenefitPlanAssumptionsUsedCalculatingNetPeriodicBenefitCostExpectedLongTermReturnOnAssetsNextFiscalYear_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Assumptions Used Calculating Net Periodic Benefit Cost Expected Long Term Return On Assets Next Fiscal Year</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DefinedBenefitPlanAssumptionsUsedCalculatingDecreaseInBasisPointExpectedLongTermReturnOnAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Decrease in expected long-term rate of return in basis points</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DefinedBenefitPlanAssumptionsUsedCalculatingDecreaseInBasisPointExpectedLongTermReturnOnAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Assumptions Used Calculating Decrease In Basis Point Expected Long Term Return On Assets</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DefinedBenefitPlanPercentageOfDecreaseInExpectedReturnOnLongTermRateOfReturn_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Decrease in percentage of expected long-term rate of return</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DefinedBenefitPlanPercentageOfDecreaseInExpectedReturnOnLongTermRateOfReturn_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Percentage Of Decrease In Expected Return On Long Term Rate Of Return</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DefinedBenefitPlanDecreaseInExpectedReturnOnLongTermRateOfReturn_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Decrease in expected long-term rate of return</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DefinedBenefitPlanDecreaseInExpectedReturnOnLongTermRateOfReturn_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Decrease In Expected Return On Long Term Rate Of Return</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_PensionAndOtherPostretirementBenefitPlansExpectedNetPeriodicBenefitCostInNextFiscalYear" xlink:type="locator" xlink:label="pbhc_PensionAndOtherPostretirementBenefitPlansExpectedNetPeriodicBenefitCostInNextFiscalYear" />
    <link:label xml:lang="en-US" xlink:label="pbhc_PensionAndOtherPostretirementBenefitPlansExpectedNetPeriodicBenefitCostInNextFiscalYear_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The expected total amount of net periodic benefit cost for defined benefit plans in the next fiscal period.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_PensionAndOtherPostretirementBenefitPlansExpectedNetPeriodicBenefitCostInNextFiscalYear" xlink:to="pbhc_PensionAndOtherPostretirementBenefitPlansExpectedNetPeriodicBenefitCostInNextFiscalYear_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanExpectedAmortizationNextFiscalYear" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanExpectedAmortizationNextFiscalYear" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanExpectedAmortizationNextFiscalYear_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Amounts that will be amortized from accumulated other comprehensive income (loss) in next fiscal year</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanExpectedAmortizationNextFiscalYear" xlink:to="us-gaap_DefinedBenefitPlanExpectedAmortizationNextFiscalYear_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanExpectedAmortizationNextFiscalYear_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Expected Amortization Next Fiscal Year</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanExpectedAmortizationOfTransitionAssetObligationNextFiscalYear" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanExpectedAmortizationOfTransitionAssetObligationNextFiscalYear" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanExpectedAmortizationOfTransitionAssetObligationNextFiscalYear_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Estimated amortization of the unrecognized transition obligation and actuarial loss in next fiscal year</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanExpectedAmortizationOfTransitionAssetObligationNextFiscalYear" xlink:to="us-gaap_DefinedBenefitPlanExpectedAmortizationOfTransitionAssetObligationNextFiscalYear_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanExpectedAmortizationOfTransitionAssetObligationNextFiscalYear_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Expected Amortization Of Transition Asset Obligation Next Fiscal Year</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_PensionAndOtherPostretirementBenefitPlansExpectedNetPeriodicBenefitCostInNextFiscalYear_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Expected net periodic benefit cost in next fiscal year</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_PensionAndOtherPostretirementBenefitPlansExpectedNetPeriodicBenefitCostInNextFiscalYear_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Pension And Other Postretirement Benefit Plans Expected Net Periodic Benefit Cost In Next Fiscal Year</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_NumberOfFundsInWhichPlanAssetsInvested" xlink:type="locator" xlink:label="pbhc_NumberOfFundsInWhichPlanAssetsInvested" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfFundsInWhichPlanAssetsInvested_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Elements refers to the number of funds in which plan assets invested.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_NumberOfFundsInWhichPlanAssetsInvested" xlink:to="pbhc_NumberOfFundsInWhichPlanAssetsInvested_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanAssetsTargetAllocationsAbstract" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanAssetsTargetAllocationsAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanAssetsTargetAllocationsAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Target Allocations [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanAssetsTargetAllocationsAbstract" xlink:to="us-gaap_DefinedBenefitPlanAssetsTargetAllocationsAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanAssetsTargetAllocationsAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Assets Target Allocations [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfFundsInWhichPlanAssetsInvested_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Number of funds in which plan assets invested</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfFundsInWhichPlanAssetsInvested_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Number Of Funds In Which Plan Assets Invested</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanPlanAssetsTargetAllocationPercentage" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanPlanAssetsTargetAllocationPercentage" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanPlanAssetsTargetAllocationPercentage_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Target plan asset allocations - well-funded</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanPlanAssetsTargetAllocationPercentage" xlink:to="us-gaap_DefinedBenefitPlanPlanAssetsTargetAllocationPercentage_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanPlanAssetsTargetAllocationPercentage_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Plan Assets Target Allocation Percentage</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_EstimatedRetirementLifeOfAssetsInTheTrust" xlink:type="locator" xlink:label="pbhc_EstimatedRetirementLifeOfAssetsInTheTrust" />
    <link:label xml:lang="en-US" xlink:label="pbhc_EstimatedRetirementLifeOfAssetsInTheTrust_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Elements refers to the Estimated retirement life of Assets in the Trust.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_EstimatedRetirementLifeOfAssetsInTheTrust" xlink:to="pbhc_EstimatedRetirementLifeOfAssetsInTheTrust_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_EstimatedRetirementLifeOfAssetsInTheTrust_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Estimated retirement life of assets in the trust</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_EstimatedRetirementLifeOfAssetsInTheTrust_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Estimated Retirement Life Of Assets In The Trust</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FairValueByFairValueHierarchyLevelAxis" xlink:type="locator" xlink:label="us-gaap_FairValueByFairValueHierarchyLevelAxis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueByFairValueHierarchyLevelAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Fair Value Hierarchy and NAV</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FairValueByFairValueHierarchyLevelAxis" xlink:to="us-gaap_FairValueByFairValueHierarchyLevelAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueByFairValueHierarchyLevelAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Fair Value By Fair Value Hierarchy Level [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FairValueMeasurementsFairValueHierarchyDomain" xlink:type="locator" xlink:label="us-gaap_FairValueMeasurementsFairValueHierarchyDomain" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueMeasurementsFairValueHierarchyDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Fair Value Hierarchy and NAV</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FairValueMeasurementsFairValueHierarchyDomain" xlink:to="us-gaap_FairValueMeasurementsFairValueHierarchyDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueMeasurementsFairValueHierarchyDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Fair Value Measurements Fair Value Hierarchy [Domain]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FairValueInputsLevel1Member" xlink:type="locator" xlink:label="us-gaap_FairValueInputsLevel1Member" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueInputsLevel1Member_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Level 1 [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FairValueInputsLevel1Member" xlink:to="us-gaap_FairValueInputsLevel1Member_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueInputsLevel1Member_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Fair Value Inputs Level1 [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FairValueInputsLevel2Member" xlink:type="locator" xlink:label="us-gaap_FairValueInputsLevel2Member" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueInputsLevel2Member_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Level 2 [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FairValueInputsLevel2Member" xlink:to="us-gaap_FairValueInputsLevel2Member_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueInputsLevel2Member_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Fair Value Inputs Level2 [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FairValueInputsLevel3Member" xlink:type="locator" xlink:label="us-gaap_FairValueInputsLevel3Member" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueInputsLevel3Member_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Level 3 [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FairValueInputsLevel3Member" xlink:to="us-gaap_FairValueInputsLevel3Member_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueInputsLevel3Member_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Fair Value Inputs Level3 [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_EquityFundsMember" xlink:type="locator" xlink:label="us-gaap_EquityFundsMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EquityFundsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Equities [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_EquityFundsMember" xlink:to="us-gaap_EquityFundsMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EquityFundsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Equity Funds [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MutualFundsEquityLargeCapValueMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Mutual Funds - Equity - Large-cap Value [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MutualFundsEquityLargeCapValueMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Mutual Funds Equity Large Cap Value [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MutualFundsEquityLargeCapGrowthMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Mutual Funds - Equity - Large-cap Growth [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MutualFundsEquityLargeCapGrowthMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Mutual Funds Equity Large Cap Growth [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MutualFundsEquityLargeCapCoreMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Mutual Funds - Equity - Large-cap Core [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MutualFundsEquityLargeCapCoreMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Mutual Funds Equity Large Cap Core [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MutualFundsEquityMidCapValueMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Mutual Funds - Equity - Mid-cap Value [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MutualFundsEquityMidCapValueMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Mutual Funds Equity Mid Cap Value [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MutualFundsEquityMidCapGrowthMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Mutual Funds - Equity - Mid-Cap Growth [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MutualFundsEquityMidCapGrowthMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Mutual Funds Equity Mid Cap Growth [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MutualFundsEquityMidCapCoreMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Mutual Funds - Equity - Mid-cap Core [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MutualFundsEquityMidCapCoreMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Mutual Funds Equity Mid Cap Core [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MutualFundsEquitySmallCapValueMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Mutual Funds - Equity - Small-cap Value [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MutualFundsEquitySmallCapValueMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Mutual Funds Equity Small Cap Value [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MutualFundsEquitySmallCapGrowthMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Mutual Funds - Equity - Small-cap Growth [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MutualFundsEquitySmallCapGrowthMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Mutual Funds Equity Small Cap Growth [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MutualFundsEquitySmallCapCoreMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Mutual Funds - Equity - Small-cap Core [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MutualFundsEquitySmallCapCoreMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Mutual Funds Equity Small Cap Core [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_InternationalEquityMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">International Equity [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_InternationalEquityMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">International Equity [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MutualFundsFixedIncomeUSCoreMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Mutual Funds Fixed Income US Core [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MutualFundsFixedIncomeUSCoreMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Mutual Funds Fixed Income U S Core [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanCashAndCashEquivalentsMember" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanCashAndCashEquivalentsMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanCashAndCashEquivalentsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Cash Equivalents-Money Market [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanCashAndCashEquivalentsMember" xlink:to="us-gaap_DefinedBenefitPlanCashAndCashEquivalentsMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanCashAndCashEquivalentsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Cash And Cash Equivalents [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanFundedStatusOfPlanAbstract" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanFundedStatusOfPlanAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanFundedStatusOfPlanAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Defined Benefit Plan, Funded (Unfunded) Status of Plan [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanFundedStatusOfPlanAbstract" xlink:to="us-gaap_DefinedBenefitPlanFundedStatusOfPlanAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanFundedStatusOfPlanAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Funded Status Of Plan [Abstract]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_NumberOfStocksHeldInPortfolioApproximate" xlink:type="locator" xlink:label="pbhc_NumberOfStocksHeldInPortfolioApproximate" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfStocksHeldInPortfolioApproximate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">This approximate number of stocks held in the portfolio.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_NumberOfStocksHeldInPortfolioApproximate" xlink:to="pbhc_NumberOfStocksHeldInPortfolioApproximate_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_DebtSecurityMaturityTerm" xlink:type="locator" xlink:label="pbhc_DebtSecurityMaturityTerm" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DebtSecurityMaturityTerm_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Period of time between issuance and maturity of debt security, in PnYnMnDTnHnMnS&apos; format, for example, &apos;P1Y5M13D&apos; represents the reported fact of one year, five months, and thirteen days.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_DebtSecurityMaturityTerm" xlink:to="pbhc_DebtSecurityMaturityTerm_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_MarketCapitalizations" xlink:type="locator" xlink:label="pbhc_MarketCapitalizations" />
    <link:label xml:lang="en-US" xlink:label="pbhc_MarketCapitalizations_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Elements refers to the percentage of market capitalization of equity security in compare to market universe.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_MarketCapitalizations" xlink:to="pbhc_MarketCapitalizations_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_PercentageOfFundAssetsInInvestmentGradeFixedIncomeSecurities" xlink:type="locator" xlink:label="pbhc_PercentageOfFundAssetsInInvestmentGradeFixedIncomeSecurities" />
    <link:label xml:lang="en-US" xlink:label="pbhc_PercentageOfFundAssetsInInvestmentGradeFixedIncomeSecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Refers to percentage of fund assets in investment grade fixed income securities.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_PercentageOfFundAssetsInInvestmentGradeFixedIncomeSecurities" xlink:to="pbhc_PercentageOfFundAssetsInInvestmentGradeFixedIncomeSecurities_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_PercentageOfTargetInvestmentsInMortgageBackedSecurities" xlink:type="locator" xlink:label="pbhc_PercentageOfTargetInvestmentsInMortgageBackedSecurities" />
    <link:label xml:lang="en-US" xlink:label="pbhc_PercentageOfTargetInvestmentsInMortgageBackedSecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Refers to the percentage of target investments in mortgage-backed securities.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_PercentageOfTargetInvestmentsInMortgageBackedSecurities" xlink:to="pbhc_PercentageOfTargetInvestmentsInMortgageBackedSecurities_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_NumberOfFundsInAssetsCategory" xlink:type="locator" xlink:label="pbhc_NumberOfFundsInAssetsCategory" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfFundsInAssetsCategory_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The number of funds invested in the asset category.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_NumberOfFundsInAssetsCategory" xlink:to="pbhc_NumberOfFundsInAssetsCategory_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_NumberOfInvestmentsInFundAssetsInInvestmentGradeFixedIncomeSecurities" xlink:type="locator" xlink:label="pbhc_NumberOfInvestmentsInFundAssetsInInvestmentGradeFixedIncomeSecurities" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfInvestmentsInFundAssetsInInvestmentGradeFixedIncomeSecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Refers to number of investments in fund assets in investment grade fixed income securities.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_NumberOfInvestmentsInFundAssetsInInvestmentGradeFixedIncomeSecurities" xlink:to="pbhc_NumberOfInvestmentsInFundAssetsInInvestmentGradeFixedIncomeSecurities_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_PercentageOfTotalAssetsInvestedInEquitySecurities" xlink:type="locator" xlink:label="pbhc_PercentageOfTotalAssetsInvestedInEquitySecurities" />
    <link:label xml:lang="en-US" xlink:label="pbhc_PercentageOfTotalAssetsInvestedInEquitySecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Percentage of total assets invested in equity securities.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_PercentageOfTotalAssetsInvestedInEquitySecurities" xlink:to="pbhc_PercentageOfTotalAssetsInvestedInEquitySecurities_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfStocksHeldInPortfolioApproximate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Approximate or typical number of stocks held in the portfolio</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfStocksHeldInPortfolioApproximate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Number Of Stocks Held In Portfolio Approximate</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MarketCapitalizations_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Market capitalizations</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MarketCapitalizations_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Market Capitalizations</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfFundsInAssetsCategory_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Assets category consist of number of funds</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfFundsInAssetsCategory_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Number Of Funds In Assets Category</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DebtSecurityMaturityTerm_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Maturity Period of debt security</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DebtSecurityMaturityTerm_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Debt Security Maturity Term</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_PercentageOfTotalAssetsInvestedInEquitySecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Percentage of total assets invested in equity securities, including common stocks, preferred stocks, and convertible securities</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_PercentageOfTotalAssetsInvestedInEquitySecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Percentage Of Total Assets Invested In Equity Securities</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_PercentageOfFundAssetsInInvestmentGradeFixedIncomeSecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Percentage of fund assets in investment grade fixed income securities</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_PercentageOfFundAssetsInInvestmentGradeFixedIncomeSecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Percentage Of Fund Assets In Investment Grade Fixed Income Securities</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfInvestmentsInFundAssetsInInvestmentGradeFixedIncomeSecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Number of investments in fund assets in investment grade fixed income securities</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfInvestmentsInFundAssetsInInvestmentGradeFixedIncomeSecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Number Of Investments In Fund Assets In Investment Grade Fixed Income Securities</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_PercentageOfTargetInvestmentsInMortgageBackedSecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Percentage of target investments in mortgage-backed securities</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_PercentageOfTargetInvestmentsInMortgageBackedSecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Percentage Of Target Investments In Mortgage Backed Securities</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanExpectedFutureEmployerContributionsNextFiscalYear" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanExpectedFutureEmployerContributionsNextFiscalYear" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanExpectedFutureEmployerContributionsNextFiscalYear_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Estimated future employer contributions in next fiscal year</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanExpectedFutureEmployerContributionsNextFiscalYear" xlink:to="us-gaap_DefinedBenefitPlanExpectedFutureEmployerContributionsNextFiscalYear_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanExpectedFutureEmployerContributionsNextFiscalYear_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Expected Future Employer Contributions Next Fiscal Year</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanEstimatedFutureBenefitPaymentsAbstract" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanEstimatedFutureBenefitPaymentsAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanEstimatedFutureBenefitPaymentsAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Expected Benefit Payments [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanEstimatedFutureBenefitPaymentsAbstract" xlink:to="us-gaap_DefinedBenefitPlanEstimatedFutureBenefitPaymentsAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanEstimatedFutureBenefitPaymentsAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Estimated Future Benefit Payments [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanExpectedFutureBenefitPaymentsNextTwelveMonths" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanExpectedFutureBenefitPaymentsNextTwelveMonths" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanExpectedFutureBenefitPaymentsNextTwelveMonths_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">2021</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanExpectedFutureBenefitPaymentsNextTwelveMonths" xlink:to="us-gaap_DefinedBenefitPlanExpectedFutureBenefitPaymentsNextTwelveMonths_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanExpectedFutureBenefitPaymentsNextTwelveMonths_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Expected Future Benefit Payments Next Twelve Months</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanExpectedFutureBenefitPaymentsYearTwo" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanExpectedFutureBenefitPaymentsYearTwo" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanExpectedFutureBenefitPaymentsYearTwo_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">2022</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanExpectedFutureBenefitPaymentsYearTwo" xlink:to="us-gaap_DefinedBenefitPlanExpectedFutureBenefitPaymentsYearTwo_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanExpectedFutureBenefitPaymentsYearTwo_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Expected Future Benefit Payments Year Two</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanExpectedFutureBenefitPaymentsYearThree" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanExpectedFutureBenefitPaymentsYearThree" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanExpectedFutureBenefitPaymentsYearThree_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">2023</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanExpectedFutureBenefitPaymentsYearThree" xlink:to="us-gaap_DefinedBenefitPlanExpectedFutureBenefitPaymentsYearThree_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanExpectedFutureBenefitPaymentsYearThree_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Expected Future Benefit Payments Year Three</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanExpectedFutureBenefitPaymentsYearFour" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanExpectedFutureBenefitPaymentsYearFour" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanExpectedFutureBenefitPaymentsYearFour_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">2024</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanExpectedFutureBenefitPaymentsYearFour" xlink:to="us-gaap_DefinedBenefitPlanExpectedFutureBenefitPaymentsYearFour_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanExpectedFutureBenefitPaymentsYearFour_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Expected Future Benefit Payments Year Four</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanExpectedFutureBenefitPaymentsYearFive" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanExpectedFutureBenefitPaymentsYearFive" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanExpectedFutureBenefitPaymentsYearFive_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">2025</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanExpectedFutureBenefitPaymentsYearFive" xlink:to="us-gaap_DefinedBenefitPlanExpectedFutureBenefitPaymentsYearFive_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanExpectedFutureBenefitPaymentsYearFive_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Expected Future Benefit Payments Year Five</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedBenefitPlanExpectedFutureBenefitPaymentsFiveFiscalYearsThereafter" xlink:type="locator" xlink:label="us-gaap_DefinedBenefitPlanExpectedFutureBenefitPaymentsFiveFiscalYearsThereafter" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanExpectedFutureBenefitPaymentsFiveFiscalYearsThereafter_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Thereafter</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedBenefitPlanExpectedFutureBenefitPaymentsFiveFiscalYearsThereafter" xlink:to="us-gaap_DefinedBenefitPlanExpectedFutureBenefitPaymentsFiveFiscalYearsThereafter_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedBenefitPlanExpectedFutureBenefitPaymentsFiveFiscalYearsThereafter_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Benefit Plan Expected Future Benefit Payments Five Fiscal Years Thereafter</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_DefinedContributionPensionAndOtherPostretirementPlansDisclosureAbstract1" xlink:type="locator" xlink:label="pbhc_DefinedContributionPensionAndOtherPostretirementPlansDisclosureAbstract1" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DefinedContributionPensionAndOtherPostretirementPlansDisclosureAbstract1_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">401 (k) Plan Abstract</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_DefinedContributionPensionAndOtherPostretirementPlansDisclosureAbstract1" xlink:to="pbhc_DefinedContributionPensionAndOtherPostretirementPlansDisclosureAbstract1_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DefinedContributionPensionAndOtherPostretirementPlansDisclosureAbstract1_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">401 (k) Plan [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DefinedContributionPensionAndOtherPostretirementPlansDisclosureAbstract1_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Contribution Pension And Other Postretirement Plans Disclosure Abstract1</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedContributionPlanCostRecognized" xlink:type="locator" xlink:label="us-gaap_DefinedContributionPlanCostRecognized" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedContributionPlanCostRecognized_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Employer contribution to 401 (k) plan</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedContributionPlanCostRecognized" xlink:to="us-gaap_DefinedContributionPlanCostRecognized_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedContributionPlanCostRecognized_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Contribution Plan Cost Recognized</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DefinedContributionPlanEmployerDiscretionaryContributionAmount" xlink:type="locator" xlink:label="us-gaap_DefinedContributionPlanEmployerDiscretionaryContributionAmount" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedContributionPlanEmployerDiscretionaryContributionAmount_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Additional safe harbor contribution</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DefinedContributionPlanEmployerDiscretionaryContributionAmount" xlink:to="us-gaap_DefinedContributionPlanEmployerDiscretionaryContributionAmount_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DefinedContributionPlanEmployerDiscretionaryContributionAmount_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Defined Contribution Plan Employer Discretionary Contribution Amount</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_DeferredCompensationArrangementWithIndividualBenefitPaymentAge" xlink:type="locator" xlink:label="pbhc_DeferredCompensationArrangementWithIndividualBenefitPaymentAge" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DeferredCompensationArrangementWithIndividualBenefitPaymentAge_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The age for the commencement of deferred compensation payments to the beneficiaries.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_DeferredCompensationArrangementWithIndividualBenefitPaymentAge" xlink:to="pbhc_DeferredCompensationArrangementWithIndividualBenefitPaymentAge_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DeferredCompensationArrangementsAbstract" xlink:type="locator" xlink:label="us-gaap_DeferredCompensationArrangementsAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredCompensationArrangementsAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Deferred Compensation Arrangements [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DeferredCompensationArrangementsAbstract" xlink:to="us-gaap_DeferredCompensationArrangementsAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredCompensationArrangementsAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deferred Compensation Arrangements [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DeferredCompensationArrangementWithIndividualBenefitPaymentAge_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Deferred compensation benefit payments age</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DeferredCompensationArrangementWithIndividualBenefitPaymentAge_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deferred Compensation Arrangement With Individual Benefit Payment Age</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DeferredCompensationArrangementWithIndividualMaximumContractualTerm1" xlink:type="locator" xlink:label="us-gaap_DeferredCompensationArrangementWithIndividualMaximumContractualTerm1" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredCompensationArrangementWithIndividualMaximumContractualTerm1_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Maximum contractual term</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DeferredCompensationArrangementWithIndividualMaximumContractualTerm1" xlink:to="us-gaap_DeferredCompensationArrangementWithIndividualMaximumContractualTerm1_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredCompensationArrangementWithIndividualMaximumContractualTerm1_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deferred Compensation Arrangement With Individual Maximum Contractual Term1</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DeferredCompensationArrangementWithIndividualRecordedLiability" xlink:type="locator" xlink:label="us-gaap_DeferredCompensationArrangementWithIndividualRecordedLiability" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredCompensationArrangementWithIndividualRecordedLiability_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Liability related to deferred compensation</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DeferredCompensationArrangementWithIndividualRecordedLiability" xlink:to="us-gaap_DeferredCompensationArrangementWithIndividualRecordedLiability_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredCompensationArrangementWithIndividualRecordedLiability_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deferred Compensation Arrangement With Individual Recorded Liability</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DeferredCompensationArrangementWithIndividualCompensationExpense" xlink:type="locator" xlink:label="us-gaap_DeferredCompensationArrangementWithIndividualCompensationExpense" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredCompensationArrangementWithIndividualCompensationExpense_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Deferred compensation expense</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DeferredCompensationArrangementWithIndividualCompensationExpense" xlink:to="us-gaap_DeferredCompensationArrangementWithIndividualCompensationExpense_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredCompensationArrangementWithIndividualCompensationExpense_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deferred Compensation Arrangement With Individual Compensation Expense</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_CashSurrenderValueOfLifeInsurance" xlink:type="locator" xlink:label="us-gaap_CashSurrenderValueOfLifeInsurance" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CashSurrenderValueOfLifeInsurance_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Cash surrender value of life insurance</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_CashSurrenderValueOfLifeInsurance" xlink:to="us-gaap_CashSurrenderValueOfLifeInsurance_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CashSurrenderValueOfLifeInsurance_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Cash Surrender Value Of Life Insurance</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_PeriodForInvoluntaryTerminationOrResignationFollowingChangeInControl" xlink:type="locator" xlink:label="pbhc_PeriodForInvoluntaryTerminationOrResignationFollowingChangeInControl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_PeriodForInvoluntaryTerminationOrResignationFollowingChangeInControl_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Represents the period for involuntary termination or resignation following a change in control after which the Company is required to make additional annual contributions to the plan.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_PeriodForInvoluntaryTerminationOrResignationFollowingChangeInControl" xlink:to="pbhc_PeriodForInvoluntaryTerminationOrResignationFollowingChangeInControl_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_PeriodForBenefitPaymentInEventOfDeathDisabilityOrTerminationFollowingChangeInControl" xlink:type="locator" xlink:label="pbhc_PeriodForBenefitPaymentInEventOfDeathDisabilityOrTerminationFollowingChangeInControl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_PeriodForBenefitPaymentInEventOfDeathDisabilityOrTerminationFollowingChangeInControl_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Represents the period for benefit payment that the Company is required to make to the executive or his beneficiary following a change in control.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_PeriodForBenefitPaymentInEventOfDeathDisabilityOrTerminationFollowingChangeInControl" xlink:to="pbhc_PeriodForBenefitPaymentInEventOfDeathDisabilityOrTerminationFollowingChangeInControl_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_NumberOfAnnualInstallmentsForBenefitPayments" xlink:type="locator" xlink:label="pbhc_NumberOfAnnualInstallmentsForBenefitPayments" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfAnnualInstallmentsForBenefitPayments_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Represents the number of annual installments for benefit payments from SERP.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_NumberOfAnnualInstallmentsForBenefitPayments" xlink:to="pbhc_NumberOfAnnualInstallmentsForBenefitPayments_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_PeriodForDeathDisabilityOrTerminationFollowingChangeInControl" xlink:type="locator" xlink:label="pbhc_PeriodForDeathDisabilityOrTerminationFollowingChangeInControl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_PeriodForDeathDisabilityOrTerminationFollowingChangeInControl_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Represents the period for death, disability or termination following a change in control after which the Company is required to make benefit payments to the executive or his beneficiary.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_PeriodForDeathDisabilityOrTerminationFollowingChangeInControl" xlink:to="pbhc_PeriodForDeathDisabilityOrTerminationFollowingChangeInControl_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_PeriodForInvoluntaryTerminationOrResignationFollowingChangeInControl_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Period for additional annual contributions following change in control</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_PeriodForInvoluntaryTerminationOrResignationFollowingChangeInControl_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Period For Involuntary Termination Or Resignation Following Change In Control</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_PeriodForDeathDisabilityOrTerminationFollowingChangeInControl_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Number of years the Bank is required to make additional annual contributions</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_PeriodForDeathDisabilityOrTerminationFollowingChangeInControl_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Period For Death Disability Or Termination Following Change In Control</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_PeriodForBenefitPaymentInEventOfDeathDisabilityOrTerminationFollowingChangeInControl_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Period for benefit payment in the event of death, disability or termination following change in control</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_PeriodForBenefitPaymentInEventOfDeathDisabilityOrTerminationFollowingChangeInControl_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Period For Benefit Payment In Event Of Death Disability Or Termination Following Change In Control</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfAnnualInstallmentsForBenefitPayments_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Number of annual installments for benefit payments</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfAnnualInstallmentsForBenefitPayments_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Number Of Annual Installments For Benefit Payments</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_PensionAndOtherPostretirementDefinedBenefitPlansLiabilitiesCurrentAndNoncurrent" xlink:type="locator" xlink:label="us-gaap_PensionAndOtherPostretirementDefinedBenefitPlansLiabilitiesCurrentAndNoncurrent" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PensionAndOtherPostretirementDefinedBenefitPlansLiabilitiesCurrentAndNoncurrent_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Other accrued liabilities</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_PensionAndOtherPostretirementDefinedBenefitPlansLiabilitiesCurrentAndNoncurrent" xlink:to="us-gaap_PensionAndOtherPostretirementDefinedBenefitPlansLiabilitiesCurrentAndNoncurrent_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PensionAndOtherPostretirementDefinedBenefitPlansLiabilitiesCurrentAndNoncurrent_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Pension And Other Postretirement Defined Benefit Plans Liabilities Current And Noncurrent</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_CommonStockConversionOfEachShare" xlink:type="locator" xlink:label="pbhc_CommonStockConversionOfEachShare" />
    <link:label xml:lang="en-US" xlink:label="pbhc_CommonStockConversionOfEachShare_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Ratio applied to the conversion of stock as part of the conversion arrangement.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_CommonStockConversionOfEachShare" xlink:to="pbhc_CommonStockConversionOfEachShare_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ScheduleOfShareBasedCompensationArrangementsByShareBasedPaymentAwardTable" xlink:type="locator" xlink:label="us-gaap_ScheduleOfShareBasedCompensationArrangementsByShareBasedPaymentAwardTable" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfShareBasedCompensationArrangementsByShareBasedPaymentAwardTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Schedule Of Share Based Compensation Arrangements By Share Based Payment Award [Table]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ScheduleOfShareBasedCompensationArrangementsByShareBasedPaymentAwardTable" xlink:to="us-gaap_ScheduleOfShareBasedCompensationArrangementsByShareBasedPaymentAwardTable_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfShareBasedCompensationArrangementsByShareBasedPaymentAwardTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Share Based Compensation Arrangements By Share Based Payment Award [Table]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_PlanNameAxis" xlink:type="locator" xlink:label="us-gaap_PlanNameAxis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PlanNameAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Plan Name</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_PlanNameAxis" xlink:to="us-gaap_PlanNameAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PlanNameAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Plan Name [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_PlanNameDomain" xlink:type="locator" xlink:label="us-gaap_PlanNameDomain" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PlanNameDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Plan Name</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_PlanNameDomain" xlink:to="us-gaap_PlanNameDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PlanNameDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Plan Name [Domain]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_April2010StockOptionGrantsMember" xlink:type="locator" xlink:label="pbhc_April2010StockOptionGrantsMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_April2010StockOptionGrantsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">April 2010 stock option grants.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_April2010StockOptionGrantsMember" xlink:to="pbhc_April2010StockOptionGrantsMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_April2010StockOptionGrantsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">April 2010 Stock Option Grants [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_April2010StockOptionGrantsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">April2010 Stock Option Grants [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AwardTypeAxis" xlink:type="locator" xlink:label="us-gaap_AwardTypeAxis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AwardTypeAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Award Type</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AwardTypeAxis" xlink:to="us-gaap_AwardTypeAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AwardTypeAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Award Type [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ShareBasedCompensationArrangementsByShareBasedPaymentAwardAwardTypeAndPlanNameDomain" xlink:type="locator" xlink:label="us-gaap_ShareBasedCompensationArrangementsByShareBasedPaymentAwardAwardTypeAndPlanNameDomain" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementsByShareBasedPaymentAwardAwardTypeAndPlanNameDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Award Type</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ShareBasedCompensationArrangementsByShareBasedPaymentAwardAwardTypeAndPlanNameDomain" xlink:to="us-gaap_ShareBasedCompensationArrangementsByShareBasedPaymentAwardAwardTypeAndPlanNameDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementsByShareBasedPaymentAwardAwardTypeAndPlanNameDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Share Based Compensation Arrangements By Share Based Payment Award Award Type And Plan Name [Domain]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/srt/2020/elts/srt-2020-01-31.xsd#srt_DirectorMember" xlink:type="locator" xlink:label="srt_DirectorMember" />
    <link:label xml:lang="en-US" xlink:label="srt_DirectorMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Directors [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="srt_DirectorMember" xlink:to="srt_DirectorMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="srt_DirectorMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Director [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/srt/2020/elts/srt-2020-01-31.xsd#srt_VicePresidentMember" xlink:type="locator" xlink:label="srt_VicePresidentMember" />
    <link:label xml:lang="en-US" xlink:label="srt_VicePresidentMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Senior Vice President [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="srt_VicePresidentMember" xlink:to="srt_VicePresidentMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="srt_VicePresidentMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Vice President [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ChiefExecutiveOfficerAndSeniorVicePresidentMember" xlink:type="locator" xlink:label="pbhc_ChiefExecutiveOfficerAndSeniorVicePresidentMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ChiefExecutiveOfficerAndSeniorVicePresidentMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Highest ranking executive officer, who has ultimate managerial responsibility for the entity and who reports to the board of directors. In addition, the chief executive officer (CEO) may also be the chairman of the board or president and rank of officer in the entity that may be appointed by the board of directors.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ChiefExecutiveOfficerAndSeniorVicePresidentMember" xlink:to="pbhc_ChiefExecutiveOfficerAndSeniorVicePresidentMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ChiefExecutiveOfficerAndSeniorVicePresidentMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Chief Executive Officer And Senior Vice President [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ChiefExecutiveOfficerAndSeniorVicePresidentMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Chief Executive Officer And Senior Vice President [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_SeniorOfficerMember" xlink:type="locator" xlink:label="pbhc_SeniorOfficerMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_SeniorOfficerMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Senior executive of the entity that is appointed to the position by the board of directors.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_SeniorOfficerMember" xlink:to="pbhc_SeniorOfficerMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_SeniorOfficerMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Senior Officer [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_SeniorOfficerMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Senior Officer [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/srt/2020/elts/srt-2020-01-31.xsd#srt_OfficerMember" xlink:type="locator" xlink:label="srt_OfficerMember" />
    <link:label xml:lang="en-US" xlink:label="srt_OfficerMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Officers [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="srt_OfficerMember" xlink:to="srt_OfficerMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="srt_OfficerMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Officer [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_May2016StockOptionGrantsMember" xlink:type="locator" xlink:label="pbhc_May2016StockOptionGrantsMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_May2016StockOptionGrantsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">May 2016 stock option grants.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_May2016StockOptionGrantsMember" xlink:to="pbhc_May2016StockOptionGrantsMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_May2016StockOptionGrantsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">May 2016 Stock Option Grants [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_May2016StockOptionGrantsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">May2016 Stock Option Grants [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_DirectorAndOfficerMember" xlink:type="locator" xlink:label="pbhc_DirectorAndOfficerMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DirectorAndOfficerMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Person serving on the board of directors (who collectively have responsibility for governing the entity). And officer is a executive of the entity that is appointed to the position by the board of directors.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_DirectorAndOfficerMember" xlink:to="pbhc_DirectorAndOfficerMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DirectorAndOfficerMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Directors and Officers [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DirectorAndOfficerMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Director And Officer [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/srt/2020/elts/srt-2020-01-31.xsd#srt_ExecutiveOfficerMember" xlink:type="locator" xlink:label="srt_ExecutiveOfficerMember" />
    <link:label xml:lang="en-US" xlink:label="srt_ExecutiveOfficerMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Executive Officers [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="srt_ExecutiveOfficerMember" xlink:to="srt_ExecutiveOfficerMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="srt_ExecutiveOfficerMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Executive Officer [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ChiefExecutiveOfficerExecutiveOfficersAndSeniorOfficerMember" xlink:type="locator" xlink:label="pbhc_ChiefExecutiveOfficerExecutiveOfficersAndSeniorOfficerMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ChiefExecutiveOfficerExecutiveOfficersAndSeniorOfficerMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Refers to Chief Executive Officer who is highest ranking executive officer, who has ultimate managerial responsibility for the entity and who reports to the board of directors, Executive officer one of the ranking officers of the entity, appointed to the position by the board of directors and also Senior executive of the entity that is appointed to the position by the board of directors.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ChiefExecutiveOfficerExecutiveOfficersAndSeniorOfficerMember" xlink:to="pbhc_ChiefExecutiveOfficerExecutiveOfficersAndSeniorOfficerMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ChiefExecutiveOfficerExecutiveOfficersAndSeniorOfficerMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Chief Executive Officer, Executive Officers and Senior Officer [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ChiefExecutiveOfficerExecutiveOfficersAndSeniorOfficerMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Chief Executive Officer Executive Officers And Senior Officer [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/srt/2020/elts/srt-2020-01-31.xsd#srt_ChiefExecutiveOfficerMember" xlink:type="locator" xlink:label="srt_ChiefExecutiveOfficerMember" />
    <link:label xml:lang="en-US" xlink:label="srt_ChiefExecutiveOfficerMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Chief Executive Officer [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="srt_ChiefExecutiveOfficerMember" xlink:to="srt_ChiefExecutiveOfficerMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="srt_ChiefExecutiveOfficerMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Chief Executive Officer [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_EmployeeIncentiveStockOptionAndNonQualifiedStockOptionMember" xlink:type="locator" xlink:label="pbhc_EmployeeIncentiveStockOptionAndNonQualifiedStockOptionMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_EmployeeIncentiveStockOptionAndNonQualifiedStockOptionMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Employee incentive stock option and non qualified stock option member.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_EmployeeIncentiveStockOptionAndNonQualifiedStockOptionMember" xlink:to="pbhc_EmployeeIncentiveStockOptionAndNonQualifiedStockOptionMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_EmployeeIncentiveStockOptionAndNonQualifiedStockOptionMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Incentive Stock Option And Non Qualified Stock Option [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_EmployeeIncentiveStockOptionAndNonQualifiedStockOptionMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Employee Incentive Stock Option And Non Qualified Stock Option [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_EmployeeIncentiveStockOptionMember" xlink:type="locator" xlink:label="pbhc_EmployeeIncentiveStockOptionMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_EmployeeIncentiveStockOptionMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Employee incentive stock option member.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_EmployeeIncentiveStockOptionMember" xlink:to="pbhc_EmployeeIncentiveStockOptionMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_EmployeeIncentiveStockOptionMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Incentive Stock Option [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_EmployeeIncentiveStockOptionMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Employee Incentive Stock Option [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_EmployeeNonQualifiedStockOptionMember" xlink:type="locator" xlink:label="pbhc_EmployeeNonQualifiedStockOptionMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_EmployeeNonQualifiedStockOptionMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Employee non qualified stock option member.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_EmployeeNonQualifiedStockOptionMember" xlink:to="pbhc_EmployeeNonQualifiedStockOptionMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_EmployeeNonQualifiedStockOptionMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Non Qualified Stock Option [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_EmployeeNonQualifiedStockOptionMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Employee Non Qualified Stock Option [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_May2016RestrictedStockUnitGrantsMember" xlink:type="locator" xlink:label="pbhc_May2016RestrictedStockUnitGrantsMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_May2016RestrictedStockUnitGrantsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">May 2016 restricted stock unit grants.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_May2016RestrictedStockUnitGrantsMember" xlink:to="pbhc_May2016RestrictedStockUnitGrantsMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_May2016RestrictedStockUnitGrantsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">May 2016 Restricted Stock Unit Grants [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_May2016RestrictedStockUnitGrantsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">May2016 Restricted Stock Unit Grants [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_RestrictedStockUnitsRSUMember" xlink:type="locator" xlink:label="us-gaap_RestrictedStockUnitsRSUMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RestrictedStockUnitsRSUMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Restricted Stock Units [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_RestrictedStockUnitsRSUMember" xlink:to="us-gaap_RestrictedStockUnitsRSUMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RestrictedStockUnitsRSUMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Restricted Stock Units R S U [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_EmployeeStockOptionAndRestrictedStockUnitsMember" xlink:type="locator" xlink:label="pbhc_EmployeeStockOptionAndRestrictedStockUnitsMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_EmployeeStockOptionAndRestrictedStockUnitsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Employee stock option and restricted stock units.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_EmployeeStockOptionAndRestrictedStockUnitsMember" xlink:to="pbhc_EmployeeStockOptionAndRestrictedStockUnitsMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_EmployeeStockOptionAndRestrictedStockUnitsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Stock Option Awards and Restricted Stock Units [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_EmployeeStockOptionAndRestrictedStockUnitsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Employee Stock Option And Restricted Stock Units [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardLineItems" xlink:type="locator" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardLineItems" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Share Based Compensation Arrangement By Share Based Payment Award [Line Items]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardLineItems" xlink:to="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardLineItems_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Share Based Compensation Arrangement By Share Based Payment Award [Line Items]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_CommonStockConversionOfEachShare_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Ratio of conversion of shares</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_CommonStockConversionOfEachShare_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Common Stock Conversion Of Each Share</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_NumberOfStockUnitsBeneficiaries" xlink:type="locator" xlink:label="pbhc_NumberOfStockUnitsBeneficiaries" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfStockUnitsBeneficiaries_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The number of stock units beneficiaries.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_NumberOfStockUnitsBeneficiaries" xlink:to="pbhc_NumberOfStockUnitsBeneficiaries_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_EmployeeStockOwnershipPlanEsopAwardVestingPeriodPercentagePerYear" xlink:type="locator" xlink:label="pbhc_EmployeeStockOwnershipPlanEsopAwardVestingPeriodPercentagePerYear" />
    <link:label xml:lang="en-US" xlink:label="pbhc_EmployeeStockOwnershipPlanEsopAwardVestingPeriodPercentagePerYear_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The annual vesting percentage of interest in the shares.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_EmployeeStockOwnershipPlanEsopAwardVestingPeriodPercentagePerYear" xlink:to="pbhc_EmployeeStockOwnershipPlanEsopAwardVestingPeriodPercentagePerYear_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardTermOfAward" xlink:type="locator" xlink:label="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardTermOfAward" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardTermOfAward_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The term of the award from the date of grant.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardTermOfAward" xlink:to="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardTermOfAward_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardNumberOfSharesAuthorized" xlink:type="locator" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardNumberOfSharesAuthorized" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardNumberOfSharesAuthorized_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Number of shares authorized (in shares)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardNumberOfSharesAuthorized" xlink:to="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardNumberOfSharesAuthorized_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardNumberOfSharesAuthorized_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Share Based Compensation Arrangement By Share Based Payment Award Number Of Shares Authorized</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfStockUnitsBeneficiaries_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Number of stock awards beneficiaries</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfStockUnitsBeneficiaries_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Number Of Stock Units Beneficiaries</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardAwardVestingPeriod1" xlink:type="locator" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardAwardVestingPeriod1" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardAwardVestingPeriod1_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Vesting period</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardAwardVestingPeriod1" xlink:to="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardAwardVestingPeriod1_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardAwardVestingPeriod1_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Share Based Compensation Arrangement By Share Based Payment Award Award Vesting Period1</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_EmployeeStockOwnershipPlanEsopAwardVestingPeriodPercentagePerYear_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Award annual vesting</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_EmployeeStockOwnershipPlanEsopAwardVestingPeriodPercentagePerYear_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Employee Stock Ownership Plan Esop Award Vesting Period Percentage Per Year</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardTermOfAward_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Term of award</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardTermOfAward_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Share Based Compensation Arrangement By Share Based Payment Award Term Of Award</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsOutstandingRollForward" xlink:type="locator" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsOutstandingRollForward" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsOutstandingRollForward_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Stock Options Outstanding [Roll Forward]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsOutstandingRollForward" xlink:to="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsOutstandingRollForward_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsOutstandingRollForward_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Share Based Compensation Arrangement By Share Based Payment Award Options Outstanding Roll Forward</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsGrantsInPeriodGross" xlink:type="locator" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsGrantsInPeriodGross" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsGrantsInPeriodGross_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Granted (in shares)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsGrantsInPeriodGross" xlink:to="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsGrantsInPeriodGross_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsGrantsInPeriodGross_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Share Based Compensation Arrangement By Share Based Payment Award Options Grants In Period Gross</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardFairValueAssumptionsAndMethodologyAbstract" xlink:type="locator" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardFairValueAssumptionsAndMethodologyAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardFairValueAssumptionsAndMethodologyAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Fair Value Assumptions [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardFairValueAssumptionsAndMethodologyAbstract" xlink:to="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardFairValueAssumptionsAndMethodologyAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardFairValueAssumptionsAndMethodologyAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Share Based Compensation Arrangement By Share Based Payment Award Fair Value Assumptions And Methodology [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardFairValueAssumptionsRiskFreeInterestRate" xlink:type="locator" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardFairValueAssumptionsRiskFreeInterestRate" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardFairValueAssumptionsRiskFreeInterestRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Risk free interest rate</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardFairValueAssumptionsRiskFreeInterestRate" xlink:to="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardFairValueAssumptionsRiskFreeInterestRate_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardFairValueAssumptionsRiskFreeInterestRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Share Based Compensation Arrangement By Share Based Payment Award Fair Value Assumptions Risk Free Interest Rate</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardFairValueAssumptionsExpectedVolatilityRate" xlink:type="locator" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardFairValueAssumptionsExpectedVolatilityRate" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardFairValueAssumptionsExpectedVolatilityRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Expected volatility rate</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardFairValueAssumptionsExpectedVolatilityRate" xlink:to="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardFairValueAssumptionsExpectedVolatilityRate_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardFairValueAssumptionsExpectedVolatilityRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Share Based Compensation Arrangement By Share Based Payment Award Fair Value Assumptions Expected Volatility Rate</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_SharebasedCompensationArrangementBySharebasedPaymentAwardFairValueAssumptionsExpectedTerm1" xlink:type="locator" xlink:label="us-gaap_SharebasedCompensationArrangementBySharebasedPaymentAwardFairValueAssumptionsExpectedTerm1" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SharebasedCompensationArrangementBySharebasedPaymentAwardFairValueAssumptionsExpectedTerm1_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Expected life</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_SharebasedCompensationArrangementBySharebasedPaymentAwardFairValueAssumptionsExpectedTerm1" xlink:to="us-gaap_SharebasedCompensationArrangementBySharebasedPaymentAwardFairValueAssumptionsExpectedTerm1_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SharebasedCompensationArrangementBySharebasedPaymentAwardFairValueAssumptionsExpectedTerm1_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Sharebased Compensation Arrangement By Sharebased Payment Award Fair Value Assumptions Expected Term1</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardFairValueAssumptionsExpectedDividendRate" xlink:type="locator" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardFairValueAssumptionsExpectedDividendRate" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardFairValueAssumptionsExpectedDividendRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Expected dividend yield</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardFairValueAssumptionsExpectedDividendRate" xlink:to="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardFairValueAssumptionsExpectedDividendRate_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardFairValueAssumptionsExpectedDividendRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Share Based Compensation Arrangement By Share Based Payment Award Fair Value Assumptions Expected Dividend Rate</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsGrantsInPeriodWeightedAverageGrantDateFairValue" xlink:type="locator" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsGrantsInPeriodWeightedAverageGrantDateFairValue" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsGrantsInPeriodWeightedAverageGrantDateFairValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Weighted average fair value of options granted (in dollars per share)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsGrantsInPeriodWeightedAverageGrantDateFairValue" xlink:to="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsGrantsInPeriodWeightedAverageGrantDateFairValue_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsGrantsInPeriodWeightedAverageGrantDateFairValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Share Based Compensation Arrangement By Share Based Payment Award Options Grants In Period Weighted Average Grant Date Fair Value</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_NumberOfBeneficiariesWhoseAwardVestUponRetirement" xlink:type="locator" xlink:label="pbhc_NumberOfBeneficiariesWhoseAwardVestUponRetirement" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfBeneficiariesWhoseAwardVestUponRetirement_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The number of beneficiaries whose award vest upon retirement.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_NumberOfBeneficiariesWhoseAwardVestUponRetirement" xlink:to="pbhc_NumberOfBeneficiariesWhoseAwardVestUponRetirement_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfBeneficiariesWhoseAwardVestUponRetirement_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Number of beneficiaries whose award vested upon retirement</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfBeneficiariesWhoseAwardVestUponRetirement_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Number Of Beneficiaries Whose Award Vest Upon Retirement</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsNewlyVested" xlink:type="locator" xlink:label="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsNewlyVested" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsNewlyVested_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Number of shares vested during the current period and available to exercise.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsNewlyVested" xlink:to="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsNewlyVested_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsOutstandingRollForward_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Stock Options Outstanding, Number of Shares [Roll Forward]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsOutstandingNumber" xlink:type="locator" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsOutstandingNumber" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsOutstandingNumber_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/periodStartLabel">Outstanding at beginning of period (in shares)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsOutstandingNumber" xlink:to="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsOutstandingNumber_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsOutstandingNumber_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Share Based Compensation Arrangement By Share Based Payment Award Options Outstanding Number</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsNewlyVested_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Newly vested (in shares)</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsNewlyVested_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Share Based Compensation Arrangement By Share Based Payment Award Options Newly Vested</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_StockIssuedDuringPeriodSharesStockOptionsExercised" xlink:type="locator" xlink:label="us-gaap_StockIssuedDuringPeriodSharesStockOptionsExercised" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_StockIssuedDuringPeriodSharesStockOptionsExercised_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Exercised (in shares)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_StockIssuedDuringPeriodSharesStockOptionsExercised" xlink:to="us-gaap_StockIssuedDuringPeriodSharesStockOptionsExercised_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_StockIssuedDuringPeriodSharesStockOptionsExercised_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Stock Issued During Period Shares Stock Options Exercised</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExpirationsInPeriod" xlink:type="locator" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExpirationsInPeriod" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExpirationsInPeriod_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Expired (in shares)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExpirationsInPeriod" xlink:to="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExpirationsInPeriod_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExpirationsInPeriod_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Share Based Compensation Arrangement By Share Based Payment Award Options Expirations In Period</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsOutstandingNumber_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/periodEndLabel">Outstanding at end of period (in shares)</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ShareBasedCompensationArrangementsByShareBasedPaymentAwardOptionsNewlyVestedInPeriodWeightedAverageExercisePrice" xlink:type="locator" xlink:label="pbhc_ShareBasedCompensationArrangementsByShareBasedPaymentAwardOptionsNewlyVestedInPeriodWeightedAverageExercisePrice" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ShareBasedCompensationArrangementsByShareBasedPaymentAwardOptionsNewlyVestedInPeriodWeightedAverageExercisePrice_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Weighted average exercise price of stock vested during the period.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ShareBasedCompensationArrangementsByShareBasedPaymentAwardOptionsNewlyVestedInPeriodWeightedAverageExercisePrice" xlink:to="pbhc_ShareBasedCompensationArrangementsByShareBasedPaymentAwardOptionsNewlyVestedInPeriodWeightedAverageExercisePrice_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsOutstandingWeightedAverageExercisePriceRollforward" xlink:type="locator" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsOutstandingWeightedAverageExercisePriceRollforward" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsOutstandingWeightedAverageExercisePriceRollforward_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Stock Options Outstanding, Weighted Average Exercise Price [Roll Forward]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsOutstandingWeightedAverageExercisePriceRollforward" xlink:to="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsOutstandingWeightedAverageExercisePriceRollforward_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsOutstandingWeightedAverageExercisePriceRollforward_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Share Based Compensation Arrangement By Share Based Payment Award Options Outstanding Weighted Average Exercise Price Rollforward</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsOutstandingWeightedAverageExercisePrice" xlink:type="locator" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsOutstandingWeightedAverageExercisePrice" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsOutstandingWeightedAverageExercisePrice_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/periodStartLabel">Outstanding at beginning of period (in dollars per share)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsOutstandingWeightedAverageExercisePrice" xlink:to="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsOutstandingWeightedAverageExercisePrice_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsOutstandingWeightedAverageExercisePrice_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Share Based Compensation Arrangement By Share Based Payment Award Options Outstanding Weighted Average Exercise Price</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ShareBasedCompensationArrangementsByShareBasedPaymentAwardOptionsGrantsInPeriodWeightedAverageExercisePrice" xlink:type="locator" xlink:label="us-gaap_ShareBasedCompensationArrangementsByShareBasedPaymentAwardOptionsGrantsInPeriodWeightedAverageExercisePrice" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementsByShareBasedPaymentAwardOptionsGrantsInPeriodWeightedAverageExercisePrice_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Granted (in dollars per share)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ShareBasedCompensationArrangementsByShareBasedPaymentAwardOptionsGrantsInPeriodWeightedAverageExercisePrice" xlink:to="us-gaap_ShareBasedCompensationArrangementsByShareBasedPaymentAwardOptionsGrantsInPeriodWeightedAverageExercisePrice_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementsByShareBasedPaymentAwardOptionsGrantsInPeriodWeightedAverageExercisePrice_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Share Based Compensation Arrangements By Share Based Payment Award Options Grants In Period Weighted Average Exercise Price</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ShareBasedCompensationArrangementsByShareBasedPaymentAwardOptionsNewlyVestedInPeriodWeightedAverageExercisePrice_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Newly vested (in dollars per share)</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ShareBasedCompensationArrangementsByShareBasedPaymentAwardOptionsNewlyVestedInPeriodWeightedAverageExercisePrice_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Share Based Compensation Arrangements By Share Based Payment Award Options Newly Vested In Period Weighted Average Exercise Price</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ShareBasedCompensationArrangementsByShareBasedPaymentAwardOptionsExercisesInPeriodWeightedAverageExercisePrice" xlink:type="locator" xlink:label="us-gaap_ShareBasedCompensationArrangementsByShareBasedPaymentAwardOptionsExercisesInPeriodWeightedAverageExercisePrice" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementsByShareBasedPaymentAwardOptionsExercisesInPeriodWeightedAverageExercisePrice_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Exercised (in dollars per share)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ShareBasedCompensationArrangementsByShareBasedPaymentAwardOptionsExercisesInPeriodWeightedAverageExercisePrice" xlink:to="us-gaap_ShareBasedCompensationArrangementsByShareBasedPaymentAwardOptionsExercisesInPeriodWeightedAverageExercisePrice_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementsByShareBasedPaymentAwardOptionsExercisesInPeriodWeightedAverageExercisePrice_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Share Based Compensation Arrangements By Share Based Payment Award Options Exercises In Period Weighted Average Exercise Price</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ShareBasedCompensationArrangementsByShareBasedPaymentAwardOptionsExpirationsInPeriodWeightedAverageExercisePrice" xlink:type="locator" xlink:label="us-gaap_ShareBasedCompensationArrangementsByShareBasedPaymentAwardOptionsExpirationsInPeriodWeightedAverageExercisePrice" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementsByShareBasedPaymentAwardOptionsExpirationsInPeriodWeightedAverageExercisePrice_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Expired (in dollars per share)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ShareBasedCompensationArrangementsByShareBasedPaymentAwardOptionsExpirationsInPeriodWeightedAverageExercisePrice" xlink:to="us-gaap_ShareBasedCompensationArrangementsByShareBasedPaymentAwardOptionsExpirationsInPeriodWeightedAverageExercisePrice_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementsByShareBasedPaymentAwardOptionsExpirationsInPeriodWeightedAverageExercisePrice_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Share Based Compensation Arrangements By Share Based Payment Award Options Expirations In Period Weighted Average Exercise Price</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsOutstandingWeightedAverageExercisePrice_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/periodEndLabel">Outstanding at end of period (in dollars per share)</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableGrantsInPeriodGross" xlink:type="locator" xlink:label="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableGrantsInPeriodGross" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableGrantsInPeriodGross_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Gross number of share options (or share units), exercisable, granted during the period.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableGrantsInPeriodGross" xlink:to="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableGrantsInPeriodGross_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableNumberOfNewlyVestedExercisableShares" xlink:type="locator" xlink:label="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableNumberOfNewlyVestedExercisableShares" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableNumberOfNewlyVestedExercisableShares_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Number of newly vested exercisable shares.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableNumberOfNewlyVestedExercisableShares" xlink:to="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableNumberOfNewlyVestedExercisableShares_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableExercisesInPeriod" xlink:type="locator" xlink:label="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableExercisesInPeriod" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableExercisesInPeriod_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Number of share options (or share units), exercisable, exercised during the current period.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableExercisesInPeriod" xlink:to="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableExercisesInPeriod_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableExpirationsInPeriod" xlink:type="locator" xlink:label="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableExpirationsInPeriod" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableExpirationsInPeriod_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Number of options or other stock instruments, exercisable, for which the right to exercise has lapsed under the terms of the plan agreements.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableExpirationsInPeriod" xlink:to="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableExpirationsInPeriod_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsAdditionalDisclosuresAbstract" xlink:type="locator" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsAdditionalDisclosuresAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsAdditionalDisclosuresAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Stock Options Shares Exercisable, Number of Shares [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsAdditionalDisclosuresAbstract" xlink:to="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsAdditionalDisclosuresAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsAdditionalDisclosuresAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Share Based Compensation Arrangement By Share Based Payment Award Options Additional Disclosures [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableNumber" xlink:type="locator" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableNumber" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableNumber_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/periodStartLabel">Options exercisable at beginning of period (in shares)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableNumber" xlink:to="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableNumber_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableNumber_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Share Based Compensation Arrangement By Share Based Payment Award Options Exercisable Number</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableGrantsInPeriodGross_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Options exercisable, Granted (in shares)</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableGrantsInPeriodGross_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Share Based Compensation Arrangement By Share Based Payment Award Options Exercisable Grants In Period Gross</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableNumberOfNewlyVestedExercisableShares_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Options exercisable, Newly vested (in shares)</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableNumberOfNewlyVestedExercisableShares_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Share Based Compensation Arrangement By Share Based Payment Award Options Exercisable Number Of Newly Vested Exercisable Shares</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableExercisesInPeriod_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Options exercisable, Exercised (in shares)</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableExercisesInPeriod_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Share Based Compensation Arrangement By Share Based Payment Award Options Exercisable Exercises In Period</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableExpirationsInPeriod_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Options exercisable, Expired (in shares)</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableExpirationsInPeriod_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Share Based Compensation Arrangement By Share Based Payment Award Options Exercisable Expirations In Period</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableNumber_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/periodEndLabel">Options exercisable at end of period (in shares)</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableGrantsInPeriodWeightedAverageExercisePrice" xlink:type="locator" xlink:label="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableGrantsInPeriodWeightedAverageExercisePrice" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableGrantsInPeriodWeightedAverageExercisePrice_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Share based compensation arrangement by share based payment award options exercisable grants in period weighted average exercise price.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableGrantsInPeriodWeightedAverageExercisePrice" xlink:to="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableGrantsInPeriodWeightedAverageExercisePrice_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableNewlyVestedInPeriodWeightedAverageExercisePrice" xlink:type="locator" xlink:label="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableNewlyVestedInPeriodWeightedAverageExercisePrice" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableNewlyVestedInPeriodWeightedAverageExercisePrice_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Share based compensation arrangement by share based payment award options exercisable newly vested in period weighted average exercise price.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableNewlyVestedInPeriodWeightedAverageExercisePrice" xlink:to="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableNewlyVestedInPeriodWeightedAverageExercisePrice_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableExercisedInPeriodWeightedAverageExercisePrice" xlink:type="locator" xlink:label="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableExercisedInPeriodWeightedAverageExercisePrice" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableExercisedInPeriodWeightedAverageExercisePrice_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Share based compensation arrangement by share based payment award options exercisable exercised in period weighted average exercise price.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableExercisedInPeriodWeightedAverageExercisePrice" xlink:to="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableExercisedInPeriodWeightedAverageExercisePrice_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableExpirationInPeriodWeightedAverageExercisePrice" xlink:type="locator" xlink:label="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableExpirationInPeriodWeightedAverageExercisePrice" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableExpirationInPeriodWeightedAverageExercisePrice_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Share based compensation arrangement by share based payment award options exercisable expiration in period weighted average exercise price.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableExpirationInPeriodWeightedAverageExercisePrice" xlink:to="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableExpirationInPeriodWeightedAverageExercisePrice_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableWeightedAverageExercisePriceRollforward" xlink:type="locator" xlink:label="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableWeightedAverageExercisePriceRollforward" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableWeightedAverageExercisePriceRollforward_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Share based compensation arrangement by share based payment award options exercisable weighted average exercise price rollforward.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableWeightedAverageExercisePriceRollforward" xlink:to="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableWeightedAverageExercisePriceRollforward_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableWeightedAverageExercisePriceRollforward_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Stock Options Shares Exercisable, Weighted Average Exercise Price [Roll Forward]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableWeightedAverageExercisePriceRollforward_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Share Based Compensation Arrangement By Share Based Payment Award Options Exercisable Weighted Average Exercise Price Rollforward</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableWeightedAverageExercisePrice" xlink:type="locator" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableWeightedAverageExercisePrice" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableWeightedAverageExercisePrice_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/periodStartLabel">Outstanding at beginning of period (in dollars per share)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableWeightedAverageExercisePrice" xlink:to="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableWeightedAverageExercisePrice_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableWeightedAverageExercisePrice_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Share Based Compensation Arrangement By Share Based Payment Award Options Exercisable Weighted Average Exercise Price</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableGrantsInPeriodWeightedAverageExercisePrice_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Granted (in dollars per share)</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableGrantsInPeriodWeightedAverageExercisePrice_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Share Based Compensation Arrangement By Share Based Payment Award Options Exercisable Grants In Period Weighted Average Exercise Price</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableNewlyVestedInPeriodWeightedAverageExercisePrice_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Newly vested (in dollars per share)</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableNewlyVestedInPeriodWeightedAverageExercisePrice_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Share Based Compensation Arrangement By Share Based Payment Award Options Exercisable Newly Vested In Period Weighted Average Exercise Price</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableExercisedInPeriodWeightedAverageExercisePrice_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Exercised (in dollars per share)</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableExercisedInPeriodWeightedAverageExercisePrice_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Share Based Compensation Arrangement By Share Based Payment Award Options Exercisable Exercised In Period Weighted Average Exercise Price</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableExpirationInPeriodWeightedAverageExercisePrice_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Expired (in dollars per share)</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableExpirationInPeriodWeightedAverageExercisePrice_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Share Based Compensation Arrangement By Share Based Payment Award Options Exercisable Expiration In Period Weighted Average Exercise Price</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisableWeightedAverageExercisePrice_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/periodEndLabel">Outstanding at end of period (in dollars per share)</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsAdditionalDisclosuresAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Options, Additional Disclosures [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisesInPeriodTotalIntrinsicValue" xlink:type="locator" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisesInPeriodTotalIntrinsicValue" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisesInPeriodTotalIntrinsicValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Intrinsic value of options exercised</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisesInPeriodTotalIntrinsicValue" xlink:to="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisesInPeriodTotalIntrinsicValue_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisesInPeriodTotalIntrinsicValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Share Based Compensation Arrangement By Share Based Payment Award Options Exercises In Period Total Intrinsic Value</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_SharebasedCompensationArrangementBySharebasedPaymentAwardOptionsOutstandingWeightedAverageRemainingContractualTerm2" xlink:type="locator" xlink:label="us-gaap_SharebasedCompensationArrangementBySharebasedPaymentAwardOptionsOutstandingWeightedAverageRemainingContractualTerm2" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SharebasedCompensationArrangementBySharebasedPaymentAwardOptionsOutstandingWeightedAverageRemainingContractualTerm2_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Weighted average remaining contractual term - options outstanding</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_SharebasedCompensationArrangementBySharebasedPaymentAwardOptionsOutstandingWeightedAverageRemainingContractualTerm2" xlink:to="us-gaap_SharebasedCompensationArrangementBySharebasedPaymentAwardOptionsOutstandingWeightedAverageRemainingContractualTerm2_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SharebasedCompensationArrangementBySharebasedPaymentAwardOptionsOutstandingWeightedAverageRemainingContractualTerm2_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Sharebased Compensation Arrangement By Sharebased Payment Award Options Outstanding Weighted Average Remaining Contractual Term2</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_SharebasedCompensationArrangementBySharebasedPaymentAwardOptionsExercisableWeightedAverageRemainingContractualTerm1" xlink:type="locator" xlink:label="us-gaap_SharebasedCompensationArrangementBySharebasedPaymentAwardOptionsExercisableWeightedAverageRemainingContractualTerm1" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SharebasedCompensationArrangementBySharebasedPaymentAwardOptionsExercisableWeightedAverageRemainingContractualTerm1_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Weighted average remaining contractual term - shares exercisable</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_SharebasedCompensationArrangementBySharebasedPaymentAwardOptionsExercisableWeightedAverageRemainingContractualTerm1" xlink:to="us-gaap_SharebasedCompensationArrangementBySharebasedPaymentAwardOptionsExercisableWeightedAverageRemainingContractualTerm1_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SharebasedCompensationArrangementBySharebasedPaymentAwardOptionsExercisableWeightedAverageRemainingContractualTerm1_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Sharebased Compensation Arrangement By Sharebased Payment Award Options Exercisable Weighted Average Remaining Contractual Term1</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardEquityInstrumentsOtherThanOptionsGrantsInPeriod" xlink:type="locator" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardEquityInstrumentsOtherThanOptionsGrantsInPeriod" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardEquityInstrumentsOtherThanOptionsGrantsInPeriod_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Granted (in shares)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardEquityInstrumentsOtherThanOptionsGrantsInPeriod" xlink:to="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardEquityInstrumentsOtherThanOptionsGrantsInPeriod_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardEquityInstrumentsOtherThanOptionsGrantsInPeriod_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Share Based Compensation Arrangement By Share Based Payment Award Equity Instruments Other Than Options Grants In Period</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_EstimatedFutureCompensationExpenseNextFiscalYear" xlink:type="locator" xlink:label="pbhc_EstimatedFutureCompensationExpenseNextFiscalYear" />
    <link:label xml:lang="en-US" xlink:label="pbhc_EstimatedFutureCompensationExpenseNextFiscalYear_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Represents the expense to be recognized during future periods arising from equity-based compensation arrangements (for example, shares of stock, unit, stock options or other equity instruments) with employees, directors and certain consultants qualifying for treatment as employees. Will be recognized in the next fiscal year.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_EstimatedFutureCompensationExpenseNextFiscalYear" xlink:to="pbhc_EstimatedFutureCompensationExpenseNextFiscalYear_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_EstimatedFutureCompensationExpenseInYearTwo" xlink:type="locator" xlink:label="pbhc_EstimatedFutureCompensationExpenseInYearTwo" />
    <link:label xml:lang="en-US" xlink:label="pbhc_EstimatedFutureCompensationExpenseInYearTwo_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Represents the expense to be recognized during future periods arising from equity-based compensation arrangements (for example, shares of stock, unit, stock options or other equity instruments) with employees, directors and certain consultants qualifying for treatment as employees. Will be recognized in the second year following the current balance sheet date.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_EstimatedFutureCompensationExpenseInYearTwo" xlink:to="pbhc_EstimatedFutureCompensationExpenseInYearTwo_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_EstimatedFutureCompensationExpenseInYearThree" xlink:type="locator" xlink:label="pbhc_EstimatedFutureCompensationExpenseInYearThree" />
    <link:label xml:lang="en-US" xlink:label="pbhc_EstimatedFutureCompensationExpenseInYearThree_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Represents the expense to be recognized during future periods arising from equity-based compensation arrangements (for example, shares of stock, unit, stock options or other equity instruments) with employees, directors and certain consultants qualifying for treatment as employees. Will be recognized in the third year following the current balance sheet date.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_EstimatedFutureCompensationExpenseInYearThree" xlink:to="pbhc_EstimatedFutureCompensationExpenseInYearThree_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_EstimatedFutureCompensationExpenseAbstract" xlink:type="locator" xlink:label="pbhc_EstimatedFutureCompensationExpenseAbstract" />
    <link:label xml:lang="en-US" xlink:label="pbhc_EstimatedFutureCompensationExpenseAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Estimated Future Compensation Expense [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_EstimatedFutureCompensationExpenseAbstract" xlink:to="pbhc_EstimatedFutureCompensationExpenseAbstract_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ShareBasedCompensation" xlink:type="locator" xlink:label="us-gaap_ShareBasedCompensation" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensation_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Compensation expense</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ShareBasedCompensation" xlink:to="us-gaap_ShareBasedCompensation_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ShareBasedCompensation_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Share Based Compensation</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_EstimatedFutureCompensationExpenseAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Estimated Future Compensation Expense [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_EstimatedFutureCompensationExpenseAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Estimated Future Compensation Expense [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_EstimatedFutureCompensationExpenseNextFiscalYear_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Estimated future compensation expense, 2021</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_EstimatedFutureCompensationExpenseNextFiscalYear_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Estimated Future Compensation Expense Next Fiscal Year</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_EstimatedFutureCompensationExpenseInYearTwo_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Estimated future compensation expense, 2022</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_EstimatedFutureCompensationExpenseInYearTwo_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Estimated Future Compensation Expense In Year Two</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_EstimatedFutureCompensationExpenseInYearThree_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Estimated future compensation expense, 2023</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_EstimatedFutureCompensationExpenseInYearThree_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Estimated Future Compensation Expense In Year Three</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_EmployeeStockOwnershipPlanEsopDebtStructureEmployerLoanTerm" xlink:type="locator" xlink:label="pbhc_EmployeeStockOwnershipPlanEsopDebtStructureEmployerLoanTerm" />
    <link:label xml:lang="en-US" xlink:label="pbhc_EmployeeStockOwnershipPlanEsopDebtStructureEmployerLoanTerm_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The term of the loan repayment.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_EmployeeStockOwnershipPlanEsopDebtStructureEmployerLoanTerm" xlink:to="pbhc_EmployeeStockOwnershipPlanEsopDebtStructureEmployerLoanTerm_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ScheduleOfEmployeeStockOwnershipPlanESOPDisclosuresTable" xlink:type="locator" xlink:label="us-gaap_ScheduleOfEmployeeStockOwnershipPlanESOPDisclosuresTable" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfEmployeeStockOwnershipPlanESOPDisclosuresTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Schedule Of Employee Stock Ownership Plan E S O P Disclosures [Table]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ScheduleOfEmployeeStockOwnershipPlanESOPDisclosuresTable" xlink:to="us-gaap_ScheduleOfEmployeeStockOwnershipPlanESOPDisclosuresTable_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfEmployeeStockOwnershipPlanESOPDisclosuresTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Employee Stock Ownership Plan E S O P Disclosures [Table]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_EmployeeStockOwnershipPlanESOPDisclosuresByPlanAxis" xlink:type="locator" xlink:label="us-gaap_EmployeeStockOwnershipPlanESOPDisclosuresByPlanAxis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EmployeeStockOwnershipPlanESOPDisclosuresByPlanAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Employee Stock Ownership Plan (ESOP) Name</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_EmployeeStockOwnershipPlanESOPDisclosuresByPlanAxis" xlink:to="us-gaap_EmployeeStockOwnershipPlanESOPDisclosuresByPlanAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EmployeeStockOwnershipPlanESOPDisclosuresByPlanAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Employee Stock Ownership Plan E S O P Disclosures By Plan [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_EmployeeStockOwnershipPlanESOPPlanDomain" xlink:type="locator" xlink:label="us-gaap_EmployeeStockOwnershipPlanESOPPlanDomain" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EmployeeStockOwnershipPlanESOPPlanDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Employee Stock Ownership Plan (ESOP), Plan</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_EmployeeStockOwnershipPlanESOPPlanDomain" xlink:to="us-gaap_EmployeeStockOwnershipPlanESOPPlanDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EmployeeStockOwnershipPlanESOPPlanDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Employee Stock Ownership Plan E S O P Plan [Domain]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_EsopMember" xlink:type="locator" xlink:label="pbhc_EsopMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_EsopMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The employee stock option plan.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_EsopMember" xlink:to="pbhc_EsopMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_EsopMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">ESOP [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_EsopMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Esop [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_EmployeeStockOwnershipPlanESOPDisclosuresLineItems" xlink:type="locator" xlink:label="us-gaap_EmployeeStockOwnershipPlanESOPDisclosuresLineItems" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EmployeeStockOwnershipPlanESOPDisclosuresLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Employee Stock Ownership Plan (ESOP) Disclosures [Line Items]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_EmployeeStockOwnershipPlanESOPDisclosuresLineItems" xlink:to="us-gaap_EmployeeStockOwnershipPlanESOPDisclosuresLineItems_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EmployeeStockOwnershipPlanESOPDisclosuresLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Employee Stock Ownership Plan E S O P Disclosures [Line Items]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_EmployeeStockOwnershipPlanESOPDebtStructureDirectLoanAmount" xlink:type="locator" xlink:label="us-gaap_EmployeeStockOwnershipPlanESOPDebtStructureDirectLoanAmount" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EmployeeStockOwnershipPlanESOPDebtStructureDirectLoanAmount_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Loan</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_EmployeeStockOwnershipPlanESOPDebtStructureDirectLoanAmount" xlink:to="us-gaap_EmployeeStockOwnershipPlanESOPDebtStructureDirectLoanAmount_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EmployeeStockOwnershipPlanESOPDebtStructureDirectLoanAmount_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Employee Stock Ownership Plan E S O P Debt Structure Direct Loan Amount</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_StockIssuedDuringPeriodSharesEmployeeStockOwnershipPlan" xlink:type="locator" xlink:label="us-gaap_StockIssuedDuringPeriodSharesEmployeeStockOwnershipPlan" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_StockIssuedDuringPeriodSharesEmployeeStockOwnershipPlan_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Shares purchased (in shares)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_StockIssuedDuringPeriodSharesEmployeeStockOwnershipPlan" xlink:to="us-gaap_StockIssuedDuringPeriodSharesEmployeeStockOwnershipPlan_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_StockIssuedDuringPeriodSharesEmployeeStockOwnershipPlan_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Stock Issued During Period Shares Employee Stock Ownership Plan</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_EmployeeStockOwnershipPlanEsopDebtStructureEmployerLoanTerm_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Term of loan repayment</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_EmployeeStockOwnershipPlanEsopDebtStructureEmployerLoanTerm_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Employee Stock Ownership Plan Esop Debt Structure Employer Loan Term</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_EmployeeStockOwnershipPlanESOPSharesAssociatedWithRefinanceLoan" xlink:type="locator" xlink:label="pbhc_EmployeeStockOwnershipPlanESOPSharesAssociatedWithRefinanceLoan" />
    <link:label xml:lang="en-US" xlink:label="pbhc_EmployeeStockOwnershipPlanESOPSharesAssociatedWithRefinanceLoan_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">This item represents the number of shares associated with the refinance loan to the Employee Stock Ownership Plan (ESOP) during the period.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_EmployeeStockOwnershipPlanESOPSharesAssociatedWithRefinanceLoan" xlink:to="pbhc_EmployeeStockOwnershipPlanESOPSharesAssociatedWithRefinanceLoan_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_EmployeeStockOwnershipPlanESOPSharesAssociatedWithNewLoan" xlink:type="locator" xlink:label="pbhc_EmployeeStockOwnershipPlanESOPSharesAssociatedWithNewLoan" />
    <link:label xml:lang="en-US" xlink:label="pbhc_EmployeeStockOwnershipPlanESOPSharesAssociatedWithNewLoan_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">This item represents the number of shares associated with new loan (including units of refinanced loan) to the Employee Stock Ownership Plan (ESOP) during the period.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_EmployeeStockOwnershipPlanESOPSharesAssociatedWithNewLoan" xlink:to="pbhc_EmployeeStockOwnershipPlanESOPSharesAssociatedWithNewLoan_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_EmployeeStockOwnershipPlanESOPSharesAssociatedWithRefinanceLoan_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Number of shares from refinanced loan (in shares)</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_EmployeeStockOwnershipPlanESOPSharesAssociatedWithRefinanceLoan_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Employee Stock Ownership Plan E S O P Shares Associated With Refinance Loan</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_EmployeeStockOwnershipPlanESOPSharesAssociatedWithNewLoan_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Number of shares from new loan (in shares)</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_EmployeeStockOwnershipPlanESOPSharesAssociatedWithNewLoan_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Employee Stock Ownership Plan E S O P Shares Associated With New Loan</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtInstrumentInterestRateStatedPercentage_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Fixed interest rate</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_EmployeeStockOwnershipPlanESOPCompensationExpense" xlink:type="locator" xlink:label="us-gaap_EmployeeStockOwnershipPlanESOPCompensationExpense" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EmployeeStockOwnershipPlanESOPCompensationExpense_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Compensation expense</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_EmployeeStockOwnershipPlanESOPCompensationExpense" xlink:to="us-gaap_EmployeeStockOwnershipPlanESOPCompensationExpense_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EmployeeStockOwnershipPlanESOPCompensationExpense_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Employee Stock Ownership Plan E S O P Compensation Expense</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_EmployeeStockOwnershipPlanESOPDividendsPaidToESOP" xlink:type="locator" xlink:label="us-gaap_EmployeeStockOwnershipPlanESOPDividendsPaidToESOP" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EmployeeStockOwnershipPlanESOPDividendsPaidToESOP_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Dividends on unallocated shares</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_EmployeeStockOwnershipPlanESOPDividendsPaidToESOP" xlink:to="us-gaap_EmployeeStockOwnershipPlanESOPDividendsPaidToESOP_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EmployeeStockOwnershipPlanESOPDividendsPaidToESOP_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Employee Stock Ownership Plan E S O P Dividends Paid To E S O P</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_EmployeeStockOwnershipPlanESOPNumberOfCommittedToBeReleasedShares" xlink:type="locator" xlink:label="us-gaap_EmployeeStockOwnershipPlanESOPNumberOfCommittedToBeReleasedShares" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EmployeeStockOwnershipPlanESOPNumberOfCommittedToBeReleasedShares_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Unearned ESOP shares (in shares)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_EmployeeStockOwnershipPlanESOPNumberOfCommittedToBeReleasedShares" xlink:to="us-gaap_EmployeeStockOwnershipPlanESOPNumberOfCommittedToBeReleasedShares_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EmployeeStockOwnershipPlanESOPNumberOfCommittedToBeReleasedShares_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Employee Stock Ownership Plan E S O P Number Of Committed To Be Released Shares</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_EmployeeStockOwnershipPlanESOPDeferredSharesFairValue" xlink:type="locator" xlink:label="us-gaap_EmployeeStockOwnershipPlanESOPDeferredSharesFairValue" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EmployeeStockOwnershipPlanESOPDeferredSharesFairValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Fair value of unearned ESOP shares</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_EmployeeStockOwnershipPlanESOPDeferredSharesFairValue" xlink:to="us-gaap_EmployeeStockOwnershipPlanESOPDeferredSharesFairValue_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EmployeeStockOwnershipPlanESOPDeferredSharesFairValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Employee Stock Ownership Plan E S O P Deferred Shares Fair Value</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_IncomeTaxExpenseBenefitContinuingOperationsAbstract" xlink:type="locator" xlink:label="us-gaap_IncomeTaxExpenseBenefitContinuingOperationsAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncomeTaxExpenseBenefitContinuingOperationsAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Income Tax Expense [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_IncomeTaxExpenseBenefitContinuingOperationsAbstract" xlink:to="us-gaap_IncomeTaxExpenseBenefitContinuingOperationsAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncomeTaxExpenseBenefitContinuingOperationsAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Income Tax Expense Benefit Continuing Operations [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_CurrentIncomeTaxExpenseBenefit" xlink:type="locator" xlink:label="us-gaap_CurrentIncomeTaxExpenseBenefit" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CurrentIncomeTaxExpenseBenefit_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Current</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_CurrentIncomeTaxExpenseBenefit" xlink:to="us-gaap_CurrentIncomeTaxExpenseBenefit_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CurrentIncomeTaxExpenseBenefit_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Current Income Tax Expense Benefit</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredIncomeTaxExpenseBenefit_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Deferred</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncomeTaxExpenseBenefit_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Income tax expense</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_IncomeTaxExpenseBenefitContinuingOperationsByJurisdictionAbstract" xlink:type="locator" xlink:label="us-gaap_IncomeTaxExpenseBenefitContinuingOperationsByJurisdictionAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncomeTaxExpenseBenefitContinuingOperationsByJurisdictionAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Income Tax Expense by Jurisdiction [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_IncomeTaxExpenseBenefitContinuingOperationsByJurisdictionAbstract" xlink:to="us-gaap_IncomeTaxExpenseBenefitContinuingOperationsByJurisdictionAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncomeTaxExpenseBenefitContinuingOperationsByJurisdictionAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Income Tax Expense Benefit Continuing Operations By Jurisdiction [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FederalIncomeTaxExpenseBenefitContinuingOperations" xlink:type="locator" xlink:label="us-gaap_FederalIncomeTaxExpenseBenefitContinuingOperations" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FederalIncomeTaxExpenseBenefitContinuingOperations_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Federal Income Tax</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FederalIncomeTaxExpenseBenefitContinuingOperations" xlink:to="us-gaap_FederalIncomeTaxExpenseBenefitContinuingOperations_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FederalIncomeTaxExpenseBenefitContinuingOperations_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Federal Income Tax Expense Benefit Continuing Operations</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_StateAndLocalIncomeTaxExpenseBenefitContinuingOperations" xlink:type="locator" xlink:label="us-gaap_StateAndLocalIncomeTaxExpenseBenefitContinuingOperations" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_StateAndLocalIncomeTaxExpenseBenefitContinuingOperations_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">State Tax</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_StateAndLocalIncomeTaxExpenseBenefitContinuingOperations" xlink:to="us-gaap_StateAndLocalIncomeTaxExpenseBenefitContinuingOperations_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_StateAndLocalIncomeTaxExpenseBenefitContinuingOperations_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">State And Local Income Tax Expense Benefit Continuing Operations</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_DeferredTaxAssetsLiabilitiesNetBeforeValuationAllowance" xlink:type="locator" xlink:label="pbhc_DeferredTaxAssetsLiabilitiesNetBeforeValuationAllowance" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DeferredTaxAssetsLiabilitiesNetBeforeValuationAllowance_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Amount before allocation of valuation allowances of deferred tax asset attributable to deductible temporary differences and carryforwards.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_DeferredTaxAssetsLiabilitiesNetBeforeValuationAllowance" xlink:to="pbhc_DeferredTaxAssetsLiabilitiesNetBeforeValuationAllowance_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DeferredTaxAssetsGrossAbstract" xlink:type="locator" xlink:label="us-gaap_DeferredTaxAssetsGrossAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredTaxAssetsGrossAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Assets:</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DeferredTaxAssetsGrossAbstract" xlink:to="us-gaap_DeferredTaxAssetsGrossAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredTaxAssetsGrossAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deferred Tax Assets Gross [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DeferredTaxAssetsTaxDeferredExpenseCompensationAndBenefitsEmployeeCompensation" xlink:type="locator" xlink:label="us-gaap_DeferredTaxAssetsTaxDeferredExpenseCompensationAndBenefitsEmployeeCompensation" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredTaxAssetsTaxDeferredExpenseCompensationAndBenefitsEmployeeCompensation_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Deferred compensation</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DeferredTaxAssetsTaxDeferredExpenseCompensationAndBenefitsEmployeeCompensation" xlink:to="us-gaap_DeferredTaxAssetsTaxDeferredExpenseCompensationAndBenefitsEmployeeCompensation_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredTaxAssetsTaxDeferredExpenseCompensationAndBenefitsEmployeeCompensation_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deferred Tax Assets Tax Deferred Expense Compensation And Benefits Employee Compensation</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DeferredTaxAssetsTaxDeferredExpenseReservesAndAccrualsProvisionForLoanLosses" xlink:type="locator" xlink:label="us-gaap_DeferredTaxAssetsTaxDeferredExpenseReservesAndAccrualsProvisionForLoanLosses" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredTaxAssetsTaxDeferredExpenseReservesAndAccrualsProvisionForLoanLosses_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Allowance for loan losses</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DeferredTaxAssetsTaxDeferredExpenseReservesAndAccrualsProvisionForLoanLosses" xlink:to="us-gaap_DeferredTaxAssetsTaxDeferredExpenseReservesAndAccrualsProvisionForLoanLosses_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredTaxAssetsTaxDeferredExpenseReservesAndAccrualsProvisionForLoanLosses_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deferred Tax Assets Tax Deferred Expense Reserves And Accruals Provision For Loan Losses</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DeferredTaxAssetsTaxDeferredExpenseCompensationAndBenefitsPostretirementBenefits" xlink:type="locator" xlink:label="us-gaap_DeferredTaxAssetsTaxDeferredExpenseCompensationAndBenefitsPostretirementBenefits" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredTaxAssetsTaxDeferredExpenseCompensationAndBenefitsPostretirementBenefits_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Postretirement benefits</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DeferredTaxAssetsTaxDeferredExpenseCompensationAndBenefitsPostretirementBenefits" xlink:to="us-gaap_DeferredTaxAssetsTaxDeferredExpenseCompensationAndBenefitsPostretirementBenefits_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredTaxAssetsTaxDeferredExpenseCompensationAndBenefitsPostretirementBenefits_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deferred Tax Assets Tax Deferred Expense Compensation And Benefits Postretirement Benefits</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DeferredTaxAssetsSubordinatedLoanInterest_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Subordinated loan interest</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DeferredTaxAssetsSubordinatedLoanInterest_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deferred Tax Assets Subordinated Loan Interest</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DeferredTaxAssetsInvestments" xlink:type="locator" xlink:label="us-gaap_DeferredTaxAssetsInvestments" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredTaxAssetsInvestments_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Investment securities</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DeferredTaxAssetsInvestments" xlink:to="us-gaap_DeferredTaxAssetsInvestments_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredTaxAssetsInvestments_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deferred Tax Assets Investments</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DeferredTaxAssetsLoanOriginationFees_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Loan origination fees</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DeferredTaxAssetsLoanOriginationFees_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deferred Tax Assets Loan Origination Fees</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DeferredTaxAssetsHeldToMaturitySecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Held-to-maturity securities</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DeferredTaxAssetsHeldToMaturitySecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deferred Tax Assets Held To Maturity Securities</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DeferredTaxAssetsTaxDeferredExpenseCompensationAndBenefitsShareBasedCompensationCost" xlink:type="locator" xlink:label="us-gaap_DeferredTaxAssetsTaxDeferredExpenseCompensationAndBenefitsShareBasedCompensationCost" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredTaxAssetsTaxDeferredExpenseCompensationAndBenefitsShareBasedCompensationCost_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Stock-based compensation</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DeferredTaxAssetsTaxDeferredExpenseCompensationAndBenefitsShareBasedCompensationCost" xlink:to="us-gaap_DeferredTaxAssetsTaxDeferredExpenseCompensationAndBenefitsShareBasedCompensationCost_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredTaxAssetsTaxDeferredExpenseCompensationAndBenefitsShareBasedCompensationCost_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deferred Tax Assets Tax Deferred Expense Compensation And Benefits Share Based Compensation Cost</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DeferredTaxAssetsCommunityServiceActivities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Community service activities</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DeferredTaxAssetsCommunityServiceActivities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deferred Tax Assets Community Service Activities</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DeferredTaxAssetsHedgingTransactions" xlink:type="locator" xlink:label="us-gaap_DeferredTaxAssetsHedgingTransactions" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredTaxAssetsHedgingTransactions_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Cash flow hedges</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DeferredTaxAssetsHedgingTransactions" xlink:to="us-gaap_DeferredTaxAssetsHedgingTransactions_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredTaxAssetsHedgingTransactions_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deferred Tax Assets Hedging Transactions</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DeferredTaxAssetsOther" xlink:type="locator" xlink:label="us-gaap_DeferredTaxAssetsOther" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredTaxAssetsOther_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Other</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DeferredTaxAssetsOther" xlink:to="us-gaap_DeferredTaxAssetsOther_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredTaxAssetsOther_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deferred Tax Assets Other</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DeferredTaxAssetsGross" xlink:type="locator" xlink:label="us-gaap_DeferredTaxAssetsGross" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredTaxAssetsGross_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Total</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DeferredTaxAssetsGross" xlink:to="us-gaap_DeferredTaxAssetsGross_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredTaxAssetsGross_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deferred Tax Assets Gross</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ComponentsOfDeferredTaxLiabilitiesAbstract" xlink:type="locator" xlink:label="us-gaap_ComponentsOfDeferredTaxLiabilitiesAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ComponentsOfDeferredTaxLiabilitiesAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Liabilities:</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ComponentsOfDeferredTaxLiabilitiesAbstract" xlink:to="us-gaap_ComponentsOfDeferredTaxLiabilitiesAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ComponentsOfDeferredTaxLiabilitiesAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Components Of Deferred Tax Liabilities [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DeferredTaxLiabilitiesPrepaidPension_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Prepaid pension</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DeferredTaxLiabilitiesPrepaidPension_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deferred Tax Liabilities Prepaid Pension</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DeferredTaxLiabilitiesInvestments" xlink:type="locator" xlink:label="us-gaap_DeferredTaxLiabilitiesInvestments" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredTaxLiabilitiesInvestments_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Investment securities</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DeferredTaxLiabilitiesInvestments" xlink:to="us-gaap_DeferredTaxLiabilitiesInvestments_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredTaxLiabilitiesInvestments_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deferred Tax Liabilities Investments</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DeferredTaxLiabilitiesPropertyPlantAndEquipment" xlink:type="locator" xlink:label="us-gaap_DeferredTaxLiabilitiesPropertyPlantAndEquipment" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredTaxLiabilitiesPropertyPlantAndEquipment_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Depreciation</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DeferredTaxLiabilitiesPropertyPlantAndEquipment" xlink:to="us-gaap_DeferredTaxLiabilitiesPropertyPlantAndEquipment_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredTaxLiabilitiesPropertyPlantAndEquipment_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deferred Tax Liabilities Property Plant And Equipment</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DeferredTaxLiabilitiesAccretion_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Accretion</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DeferredTaxLiabilitiesAccretion_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deferred Tax Liabilities Accretion</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DeferredTaxLiabilitiesGoodwillAndIntangibleAssets" xlink:type="locator" xlink:label="us-gaap_DeferredTaxLiabilitiesGoodwillAndIntangibleAssets" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredTaxLiabilitiesGoodwillAndIntangibleAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Intangible assets</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DeferredTaxLiabilitiesGoodwillAndIntangibleAssets" xlink:to="us-gaap_DeferredTaxLiabilitiesGoodwillAndIntangibleAssets_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredTaxLiabilitiesGoodwillAndIntangibleAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deferred Tax Liabilities Goodwill And Intangible Assets</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DeferredTaxLiabilitiesMortgageServicingRights" xlink:type="locator" xlink:label="us-gaap_DeferredTaxLiabilitiesMortgageServicingRights" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredTaxLiabilitiesMortgageServicingRights_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Mortgage servicing rights</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DeferredTaxLiabilitiesMortgageServicingRights" xlink:to="us-gaap_DeferredTaxLiabilitiesMortgageServicingRights_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredTaxLiabilitiesMortgageServicingRights_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deferred Tax Liabilities Mortgage Servicing Rights</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DeferredTaxLiabilitiesPrepaidExpenses" xlink:type="locator" xlink:label="us-gaap_DeferredTaxLiabilitiesPrepaidExpenses" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredTaxLiabilitiesPrepaidExpenses_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Prepaid expenses and transaction fees</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DeferredTaxLiabilitiesPrepaidExpenses" xlink:to="us-gaap_DeferredTaxLiabilitiesPrepaidExpenses_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredTaxLiabilitiesPrepaidExpenses_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deferred Tax Liabilities Prepaid Expenses</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DeferredTaxLiabilitiesLoanOriginationFees_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Loan origination fees</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DeferredTaxLiabilitiesLoanOriginationFees_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deferred Tax Liabilities Loan Origination Fees</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DeferredIncomeTaxLiabilities" xlink:type="locator" xlink:label="us-gaap_DeferredIncomeTaxLiabilities" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredIncomeTaxLiabilities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedTotalLabel">Total</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DeferredIncomeTaxLiabilities" xlink:to="us-gaap_DeferredIncomeTaxLiabilities_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredIncomeTaxLiabilities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deferred Income Tax Liabilities</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DeferredTaxAssetsLiabilitiesNetBeforeValuationAllowance_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Deferred tax assets liabilities, net before valuation allowance</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DeferredTaxAssetsLiabilitiesNetBeforeValuationAllowance_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deferred Tax Assets Liabilities Net Before Valuation Allowance</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DeferredTaxAssetsValuationAllowance" xlink:type="locator" xlink:label="us-gaap_DeferredTaxAssetsValuationAllowance" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredTaxAssetsValuationAllowance_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Less: deferred tax asset valuation allowance</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DeferredTaxAssetsValuationAllowance" xlink:to="us-gaap_DeferredTaxAssetsValuationAllowance_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredTaxAssetsValuationAllowance_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deferred Tax Assets Valuation Allowance</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DeferredTaxLiabilities" xlink:type="locator" xlink:label="us-gaap_DeferredTaxLiabilities" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredTaxLiabilities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedTotalLabel">Net deferred tax asset (liability)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DeferredTaxLiabilities" xlink:to="us-gaap_DeferredTaxLiabilities_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredTaxLiabilities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deferred Tax Liabilities</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DeferredTaxAssetsLiabilitiesNet" xlink:type="locator" xlink:label="us-gaap_DeferredTaxAssetsLiabilitiesNet" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredTaxAssetsLiabilitiesNet_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Net deferred tax asset (liability)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DeferredTaxAssetsLiabilitiesNet" xlink:to="us-gaap_DeferredTaxAssetsLiabilitiesNet_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredTaxAssetsLiabilitiesNet_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deferred Tax Assets Liabilities Net</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ModificationsAvailableToCommunityBanksForMaximumAmountOfAssetsOfBanksRegardingDeductionsInInterestIncome" xlink:type="locator" xlink:label="pbhc_ModificationsAvailableToCommunityBanksForMaximumAmountOfAssetsOfBanksRegardingDeductionsInInterestIncome" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ModificationsAvailableToCommunityBanksForMaximumAmountOfAssetsOfBanksRegardingDeductionsInInterestIncome_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Modifications available to community banks for maximum amount of assets of banks regarding deductions in interest income.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ModificationsAvailableToCommunityBanksForMaximumAmountOfAssetsOfBanksRegardingDeductionsInInterestIncome" xlink:to="pbhc_ModificationsAvailableToCommunityBanksForMaximumAmountOfAssetsOfBanksRegardingDeductionsInInterestIncome_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_IncomeTaxesLineItems" xlink:type="locator" xlink:label="pbhc_IncomeTaxesLineItems" />
    <link:label xml:lang="en-US" xlink:label="pbhc_IncomeTaxesLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Income Taxes [Line Items].</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_IncomeTaxesLineItems" xlink:to="pbhc_IncomeTaxesLineItems_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_IncomeTaxesTable" xlink:type="locator" xlink:label="pbhc_IncomeTaxesTable" />
    <link:label xml:lang="en-US" xlink:label="pbhc_IncomeTaxesTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Income Taxes [Table].</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_IncomeTaxesTable" xlink:to="pbhc_IncomeTaxesTable_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_IncomeTaxesTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Income Taxes [Table]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_IncomeTaxesTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Income Taxes [Table]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_IncomeTaxAuthorityNameAxis" xlink:type="locator" xlink:label="us-gaap_IncomeTaxAuthorityNameAxis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncomeTaxAuthorityNameAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Income Tax Authority, Name</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_IncomeTaxAuthorityNameAxis" xlink:to="us-gaap_IncomeTaxAuthorityNameAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncomeTaxAuthorityNameAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Income Tax Authority Name [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_IncomeTaxAuthorityNameDomain" xlink:type="locator" xlink:label="us-gaap_IncomeTaxAuthorityNameDomain" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncomeTaxAuthorityNameDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Income Tax Authority, Name</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_IncomeTaxAuthorityNameDomain" xlink:to="us-gaap_IncomeTaxAuthorityNameDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncomeTaxAuthorityNameDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Income Tax Authority Name [Domain]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_NewYorkStateDivisionOfTaxationAndFinanceMember" xlink:type="locator" xlink:label="us-gaap_NewYorkStateDivisionOfTaxationAndFinanceMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_NewYorkStateDivisionOfTaxationAndFinanceMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">New York [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_NewYorkStateDivisionOfTaxationAndFinanceMember" xlink:to="us-gaap_NewYorkStateDivisionOfTaxationAndFinanceMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_NewYorkStateDivisionOfTaxationAndFinanceMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">New York State Division Of Taxation And Finance [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_CarrybackYearsMember" xlink:type="locator" xlink:label="pbhc_CarrybackYearsMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_CarrybackYearsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Carryback years.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_CarrybackYearsMember" xlink:to="pbhc_CarrybackYearsMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_CarrybackYearsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Carryback Years [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_CarrybackYearsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Carryback Years [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_SubsequentEventTypeAxis" xlink:type="locator" xlink:label="us-gaap_SubsequentEventTypeAxis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SubsequentEventTypeAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Subsequent Event Type</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_SubsequentEventTypeAxis" xlink:to="us-gaap_SubsequentEventTypeAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SubsequentEventTypeAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Subsequent Event Type [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_SubsequentEventTypeDomain" xlink:type="locator" xlink:label="us-gaap_SubsequentEventTypeDomain" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SubsequentEventTypeDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Subsequent Event Type</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_SubsequentEventTypeDomain" xlink:to="us-gaap_SubsequentEventTypeDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SubsequentEventTypeDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Subsequent Event Type [Domain]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_SubsequentEventMember" xlink:type="locator" xlink:label="us-gaap_SubsequentEventMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SubsequentEventMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Subsequent Event [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_SubsequentEventMember" xlink:to="us-gaap_SubsequentEventMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SubsequentEventMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Subsequent Event [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_IncomeTaxesLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Income Taxes [Line Items]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_IncomeTaxesLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Income Taxes [Line Items]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_EffectiveIncomeTaxRateReconciliationStateAndLocalIncomeTaxes" xlink:type="locator" xlink:label="us-gaap_EffectiveIncomeTaxRateReconciliationStateAndLocalIncomeTaxes" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EffectiveIncomeTaxRateReconciliationStateAndLocalIncomeTaxes_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">State tax law, rate</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_EffectiveIncomeTaxRateReconciliationStateAndLocalIncomeTaxes" xlink:to="us-gaap_EffectiveIncomeTaxRateReconciliationStateAndLocalIncomeTaxes_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EffectiveIncomeTaxRateReconciliationStateAndLocalIncomeTaxes_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Effective Income Tax Rate Reconciliation State And Local Income Taxes</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ModificationsAvailableToCommunityBanksForMaximumAmountOfAssetsOfBanksRegardingDeductionsInInterestIncome_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Modifications available to community banks for maximum amount of assets of banks regarding deductions in interest income</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ModificationsAvailableToCommunityBanksForMaximumAmountOfAssetsOfBanksRegardingDeductionsInInterestIncome_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Modifications Available To Community Banks For Maximum Amount Of Assets Of Banks Regarding Deductions In Interest Income</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_DeferredTaxAssetsValuationAllowance_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Deferred tax valuation allowance</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_EffectiveIncomeTaxRateReconciliationChangeInEnactedTaxRate" xlink:type="locator" xlink:label="us-gaap_EffectiveIncomeTaxRateReconciliationChangeInEnactedTaxRate" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EffectiveIncomeTaxRateReconciliationChangeInEnactedTaxRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Increase in company&apos;s effective tax rate</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_EffectiveIncomeTaxRateReconciliationChangeInEnactedTaxRate" xlink:to="us-gaap_EffectiveIncomeTaxRateReconciliationChangeInEnactedTaxRate_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EffectiveIncomeTaxRateReconciliationChangeInEnactedTaxRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Effective Income Tax Rate Reconciliation Change In Enacted Tax Rate</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_IncreaseDecreaseInDeferredIncomeTaxes" xlink:type="locator" xlink:label="us-gaap_IncreaseDecreaseInDeferredIncomeTaxes" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncreaseDecreaseInDeferredIncomeTaxes_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Deferred Income Taxes</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_IncreaseDecreaseInDeferredIncomeTaxes" xlink:to="us-gaap_IncreaseDecreaseInDeferredIncomeTaxes_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncreaseDecreaseInDeferredIncomeTaxes_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Increase Decrease In Deferred Income Taxes</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_EffectiveIncomeTaxRateContinuingOperations" xlink:type="locator" xlink:label="us-gaap_EffectiveIncomeTaxRateContinuingOperations" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EffectiveIncomeTaxRateContinuingOperations_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Effective income tax rate</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_EffectiveIncomeTaxRateContinuingOperations" xlink:to="us-gaap_EffectiveIncomeTaxRateContinuingOperations_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EffectiveIncomeTaxRateContinuingOperations_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Effective Income Tax Rate Continuing Operations</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ProceedsFromIncomeTaxRefunds" xlink:type="locator" xlink:label="us-gaap_ProceedsFromIncomeTaxRefunds" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ProceedsFromIncomeTaxRefunds_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Income tax refunds</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ProceedsFromIncomeTaxRefunds" xlink:to="us-gaap_ProceedsFromIncomeTaxRefunds_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ProceedsFromIncomeTaxRefunds_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Proceeds From Income Tax Refunds</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_EffectiveIncomeTaxRateReconciliationIncreaseInValueOfBankOwnedLifeInsuranceLessPremiumPaid" xlink:type="locator" xlink:label="pbhc_EffectiveIncomeTaxRateReconciliationIncreaseInValueOfBankOwnedLifeInsuranceLessPremiumPaid" />
    <link:label xml:lang="en-US" xlink:label="pbhc_EffectiveIncomeTaxRateReconciliationIncreaseInValueOfBankOwnedLifeInsuranceLessPremiumPaid_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The portion of the difference between the effective income tax rate and domestic federal statutory income tax rate attributable to the increase in the value of bank owned life insurance less premium paid.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_EffectiveIncomeTaxRateReconciliationIncreaseInValueOfBankOwnedLifeInsuranceLessPremiumPaid" xlink:to="pbhc_EffectiveIncomeTaxRateReconciliationIncreaseInValueOfBankOwnedLifeInsuranceLessPremiumPaid_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_EffectiveIncomeTaxRateReconciliationNetOperatingLossCarryBackReceivableNetOfFederalBenfit" xlink:type="locator" xlink:label="pbhc_EffectiveIncomeTaxRateReconciliationNetOperatingLossCarryBackReceivableNetOfFederalBenfit" />
    <link:label xml:lang="en-US" xlink:label="pbhc_EffectiveIncomeTaxRateReconciliationNetOperatingLossCarryBackReceivableNetOfFederalBenfit_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Effective Income Tax Rate Reconciliation net operating loss carry back receivable, net of federal benfit.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_EffectiveIncomeTaxRateReconciliationNetOperatingLossCarryBackReceivableNetOfFederalBenfit" xlink:to="pbhc_EffectiveIncomeTaxRateReconciliationNetOperatingLossCarryBackReceivableNetOfFederalBenfit_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_IncomeTaxExaminationTable" xlink:type="locator" xlink:label="us-gaap_IncomeTaxExaminationTable" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncomeTaxExaminationTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Income Tax Examination [Table]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_IncomeTaxExaminationTable" xlink:to="us-gaap_IncomeTaxExaminationTable_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncomeTaxExaminationTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Income Tax Examination [Table]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_IncomeTaxExaminationLineItems" xlink:type="locator" xlink:label="us-gaap_IncomeTaxExaminationLineItems" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncomeTaxExaminationLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Income Tax Examination [Line Items]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_IncomeTaxExaminationLineItems" xlink:to="us-gaap_IncomeTaxExaminationLineItems_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncomeTaxExaminationLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Income Tax Examination [Line Items]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_EffectiveIncomeTaxRateContinuingOperationsTaxRateReconciliationAbstract" xlink:type="locator" xlink:label="us-gaap_EffectiveIncomeTaxRateContinuingOperationsTaxRateReconciliationAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EffectiveIncomeTaxRateContinuingOperationsTaxRateReconciliationAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Reconciliation of the Federal Statutory Income Tax Rate to the Effective Income Tax Rate [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_EffectiveIncomeTaxRateContinuingOperationsTaxRateReconciliationAbstract" xlink:to="us-gaap_EffectiveIncomeTaxRateContinuingOperationsTaxRateReconciliationAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EffectiveIncomeTaxRateContinuingOperationsTaxRateReconciliationAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Effective Income Tax Rate Continuing Operations Tax Rate Reconciliation [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_EffectiveIncomeTaxRateReconciliationAtFederalStatutoryIncomeTaxRate" xlink:type="locator" xlink:label="us-gaap_EffectiveIncomeTaxRateReconciliationAtFederalStatutoryIncomeTaxRate" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EffectiveIncomeTaxRateReconciliationAtFederalStatutoryIncomeTaxRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Federal statutory income tax rate</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_EffectiveIncomeTaxRateReconciliationAtFederalStatutoryIncomeTaxRate" xlink:to="us-gaap_EffectiveIncomeTaxRateReconciliationAtFederalStatutoryIncomeTaxRate_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EffectiveIncomeTaxRateReconciliationAtFederalStatutoryIncomeTaxRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Effective Income Tax Rate Reconciliation At Federal Statutory Income Tax Rate</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_EffectiveIncomeTaxRateReconciliationStateAndLocalIncomeTaxes_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">State tax, net of federal benefit</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_EffectiveIncomeTaxRateReconciliationTaxExemptIncome" xlink:type="locator" xlink:label="us-gaap_EffectiveIncomeTaxRateReconciliationTaxExemptIncome" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EffectiveIncomeTaxRateReconciliationTaxExemptIncome_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Tax-exempt interest income</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_EffectiveIncomeTaxRateReconciliationTaxExemptIncome" xlink:to="us-gaap_EffectiveIncomeTaxRateReconciliationTaxExemptIncome_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EffectiveIncomeTaxRateReconciliationTaxExemptIncome_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Effective Income Tax Rate Reconciliation Tax Exempt Income</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_EffectiveIncomeTaxRateReconciliationIncreaseInValueOfBankOwnedLifeInsuranceLessPremiumPaid_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Increase in value of bank owned life insurance less premiums paid</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_EffectiveIncomeTaxRateReconciliationIncreaseInValueOfBankOwnedLifeInsuranceLessPremiumPaid_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Effective Income Tax Rate Reconciliation Increase In Value Of Bank Owned Life Insurance Less Premium Paid</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_EffectiveIncomeTaxRateReconciliationChangeInDeferredTaxAssetsValuationAllowance" xlink:type="locator" xlink:label="us-gaap_EffectiveIncomeTaxRateReconciliationChangeInDeferredTaxAssetsValuationAllowance" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EffectiveIncomeTaxRateReconciliationChangeInDeferredTaxAssetsValuationAllowance_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Change in valuation allowance</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_EffectiveIncomeTaxRateReconciliationChangeInDeferredTaxAssetsValuationAllowance" xlink:to="us-gaap_EffectiveIncomeTaxRateReconciliationChangeInDeferredTaxAssetsValuationAllowance_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EffectiveIncomeTaxRateReconciliationChangeInDeferredTaxAssetsValuationAllowance_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Effective Income Tax Rate Reconciliation Change In Deferred Tax Assets Valuation Allowance</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_EffectiveIncomeTaxRateReconciliationNetOperatingLossCarryBackReceivableNetOfFederalBenfit_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">NYS net operating loss carryback filing receivable, net of federal benefit</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_EffectiveIncomeTaxRateReconciliationNetOperatingLossCarryBackReceivableNetOfFederalBenfit_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Effective Income Tax Rate Reconciliation Net Operating Loss Carry Back Receivable Net Of Federal Benfit</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_EffectiveIncomeTaxRateReconciliationOtherAdjustments" xlink:type="locator" xlink:label="us-gaap_EffectiveIncomeTaxRateReconciliationOtherAdjustments" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EffectiveIncomeTaxRateReconciliationOtherAdjustments_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Other</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_EffectiveIncomeTaxRateReconciliationOtherAdjustments" xlink:to="us-gaap_EffectiveIncomeTaxRateReconciliationOtherAdjustments_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EffectiveIncomeTaxRateReconciliationOtherAdjustments_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Effective Income Tax Rate Reconciliation Other Adjustments</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_EffectiveIncomeTaxRateContinuingOperations_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Effective income tax rate</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_EffectiveIncomeTaxRateReconciliationMinorityInterestIncomeExpense" xlink:type="locator" xlink:label="us-gaap_EffectiveIncomeTaxRateReconciliationMinorityInterestIncomeExpense" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EffectiveIncomeTaxRateReconciliationMinorityInterestIncomeExpense_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Minority interest</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_EffectiveIncomeTaxRateReconciliationMinorityInterestIncomeExpense" xlink:to="us-gaap_EffectiveIncomeTaxRateReconciliationMinorityInterestIncomeExpense_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EffectiveIncomeTaxRateReconciliationMinorityInterestIncomeExpense_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Effective Income Tax Rate Reconciliation Minority Interest Income Expense</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ScheduleOfFairValueOffBalanceSheetRisksTable" xlink:type="locator" xlink:label="us-gaap_ScheduleOfFairValueOffBalanceSheetRisksTable" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfFairValueOffBalanceSheetRisksTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Schedule Of Fair Value Off Balance Sheet Risks [Table]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ScheduleOfFairValueOffBalanceSheetRisksTable" xlink:to="us-gaap_ScheduleOfFairValueOffBalanceSheetRisksTable_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfFairValueOffBalanceSheetRisksTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Fair Value Off Balance Sheet Risks [Table]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_CommitmentsToExtendCreditMember" xlink:type="locator" xlink:label="us-gaap_CommitmentsToExtendCreditMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CommitmentsToExtendCreditMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Commitments to Grant Loans [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_CommitmentsToExtendCreditMember" xlink:to="us-gaap_CommitmentsToExtendCreditMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CommitmentsToExtendCreditMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Commitments To Extend Credit [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_UnfundedCommitmentsUnderLinesOfCreditMember" xlink:type="locator" xlink:label="pbhc_UnfundedCommitmentsUnderLinesOfCreditMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_UnfundedCommitmentsUnderLinesOfCreditMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Commitment to lend money under existing lines of credit to customers in good standing.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_UnfundedCommitmentsUnderLinesOfCreditMember" xlink:to="pbhc_UnfundedCommitmentsUnderLinesOfCreditMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_UnfundedCommitmentsUnderLinesOfCreditMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Unfunded Commitments Under Lines Of Credit [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_UnfundedCommitmentsUnderLinesOfCreditMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Unfunded Commitments Under Lines Of Credit [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_UnfundedCommitmentsRelatedToConstructionLoansInProgressMember" xlink:type="locator" xlink:label="pbhc_UnfundedCommitmentsRelatedToConstructionLoansInProgressMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_UnfundedCommitmentsRelatedToConstructionLoansInProgressMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Commitment to lend money under unfunded commitments related to construction loans in progress.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_UnfundedCommitmentsRelatedToConstructionLoansInProgressMember" xlink:to="pbhc_UnfundedCommitmentsRelatedToConstructionLoansInProgressMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_UnfundedCommitmentsRelatedToConstructionLoansInProgressMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Unfunded Commitments Related to Construction Loans in Progress [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_UnfundedCommitmentsRelatedToConstructionLoansInProgressMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Unfunded Commitments Related To Construction Loans In Progress [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_StandbyLettersOfCreditMember" xlink:type="locator" xlink:label="us-gaap_StandbyLettersOfCreditMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_StandbyLettersOfCreditMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Standby Letters of Credit [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_StandbyLettersOfCreditMember" xlink:to="us-gaap_StandbyLettersOfCreditMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_StandbyLettersOfCreditMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Standby Letters Of Credit [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FairValueOffBalanceSheetRisksDisclosureInformationLineItems" xlink:type="locator" xlink:label="us-gaap_FairValueOffBalanceSheetRisksDisclosureInformationLineItems" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueOffBalanceSheetRisksDisclosureInformationLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Fair Value, Off-balance Sheet Risks, Disclosure Information [Line Items]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FairValueOffBalanceSheetRisksDisclosureInformationLineItems" xlink:to="us-gaap_FairValueOffBalanceSheetRisksDisclosureInformationLineItems_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueOffBalanceSheetRisksDisclosureInformationLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Fair Value Off Balance Sheet Risks Disclosure Information [Line Items]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ContractualObligation" xlink:type="locator" xlink:label="us-gaap_ContractualObligation" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ContractualObligation_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Contractual obligation</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ContractualObligation" xlink:to="us-gaap_ContractualObligation_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ContractualObligation_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Contractual Obligation</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_TermOfLettersOfCreditMaximum" xlink:type="locator" xlink:label="pbhc_TermOfLettersOfCreditMaximum" />
    <link:label xml:lang="en-US" xlink:label="pbhc_TermOfLettersOfCreditMaximum_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The maximum term of letters of credit.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_TermOfLettersOfCreditMaximum" xlink:to="pbhc_TermOfLettersOfCreditMaximum_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_LoanCommitmentsOutstandingWithVariableInterestRates" xlink:type="locator" xlink:label="pbhc_LoanCommitmentsOutstandingWithVariableInterestRates" />
    <link:label xml:lang="en-US" xlink:label="pbhc_LoanCommitmentsOutstandingWithVariableInterestRates_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Loan commitments outstanding with variable interest rates.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_LoanCommitmentsOutstandingWithVariableInterestRates" xlink:to="pbhc_LoanCommitmentsOutstandingWithVariableInterestRates_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_LoanCommitmentsOutstandingWithFixedInterestRates" xlink:type="locator" xlink:label="pbhc_LoanCommitmentsOutstandingWithFixedInterestRates" />
    <link:label xml:lang="en-US" xlink:label="pbhc_LoanCommitmentsOutstandingWithFixedInterestRates_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Amount of contractual obligation, including but not limited to, long-term debt, capital lease obligations, operating lease obligations, purchase obligations, and other commitments.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_LoanCommitmentsOutstandingWithFixedInterestRates" xlink:to="pbhc_LoanCommitmentsOutstandingWithFixedInterestRates_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_LoanCommitmentsOutstandingWithVariableInterestRates_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Loan commitments outstanding with variable interest rates</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_LoanCommitmentsOutstandingWithVariableInterestRates_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Loan Commitments Outstanding With Variable Interest Rates</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_LoanCommitmentsOutstandingWithFixedInterestRates_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Loan commitments outstanding with fixed interest rates</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_LoanCommitmentsOutstandingWithFixedInterestRates_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Loan Commitments Outstanding With Fixed Interest Rates</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_TermOfLettersOfCreditMaximum_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Term of letters of credit, maximum</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_TermOfLettersOfCreditMaximum_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Term Of Letters Of Credit Maximum</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ScheduleOfStockByClassTable" xlink:type="locator" xlink:label="us-gaap_ScheduleOfStockByClassTable" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfStockByClassTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Schedule Of Stock By Class [Table]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ScheduleOfStockByClassTable" xlink:to="us-gaap_ScheduleOfStockByClassTable_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfStockByClassTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Stock By Class [Table]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_UnusualOrInfrequentItemAxis" xlink:type="locator" xlink:label="us-gaap_UnusualOrInfrequentItemAxis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_UnusualOrInfrequentItemAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Unusual or Infrequent Item, or Both</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_UnusualOrInfrequentItemAxis" xlink:to="us-gaap_UnusualOrInfrequentItemAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_UnusualOrInfrequentItemAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Unusual Or Infrequent Item [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_UnusualOrInfrequentItemDomain" xlink:type="locator" xlink:label="us-gaap_UnusualOrInfrequentItemDomain" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_UnusualOrInfrequentItemDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Unusual or Infrequent Item, or Both</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_UnusualOrInfrequentItemDomain" xlink:to="us-gaap_UnusualOrInfrequentItemDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_UnusualOrInfrequentItemDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Unusual Or Infrequent Item [Domain]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_CARESActMember" xlink:type="locator" xlink:label="pbhc_CARESActMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_CARESActMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">CARES act member</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_CARESActMember" xlink:to="pbhc_CARESActMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_CARESActMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">CARES Act</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_CARESActMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">C A R E S Act [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ClassOfStockLineItems" xlink:type="locator" xlink:label="us-gaap_ClassOfStockLineItems" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ClassOfStockLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Class Of Stock [Line Items]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ClassOfStockLineItems" xlink:to="us-gaap_ClassOfStockLineItems_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ClassOfStockLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Class Of Stock [Line Items]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AmountAvailableForDividendDistributionWithoutAffectingCapitalAdequacyRequirements" xlink:type="locator" xlink:label="us-gaap_AmountAvailableForDividendDistributionWithoutAffectingCapitalAdequacyRequirements" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AmountAvailableForDividendDistributionWithoutAffectingCapitalAdequacyRequirements_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Retained earnings legally available to pay dividends</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AmountAvailableForDividendDistributionWithoutAffectingCapitalAdequacyRequirements" xlink:to="us-gaap_AmountAvailableForDividendDistributionWithoutAffectingCapitalAdequacyRequirements_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AmountAvailableForDividendDistributionWithoutAffectingCapitalAdequacyRequirements_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Amount Available For Dividend Distribution Without Affecting Capital Adequacy Requirements</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ProceedsFromDividendsReceived" xlink:type="locator" xlink:label="us-gaap_ProceedsFromDividendsReceived" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ProceedsFromDividendsReceived_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Proceeds from dividends received</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ProceedsFromDividendsReceived" xlink:to="us-gaap_ProceedsFromDividendsReceived_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ProceedsFromDividendsReceived_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Proceeds From Dividends Received</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_CommunityBankLeverageRatioAmount" xlink:type="locator" xlink:label="pbhc_CommunityBankLeverageRatioAmount" />
    <link:label xml:lang="en-US" xlink:label="pbhc_CommunityBankLeverageRatioAmount_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Community bank leverage ratio amount.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_CommunityBankLeverageRatioAmount" xlink:to="pbhc_CommunityBankLeverageRatioAmount_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_TierOneLeverageCapitalToAverageAssets" xlink:type="locator" xlink:label="us-gaap_TierOneLeverageCapitalToAverageAssets" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TierOneLeverageCapitalToAverageAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Community bank leverage ratio</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_TierOneLeverageCapitalToAverageAssets" xlink:to="us-gaap_TierOneLeverageCapitalToAverageAssets_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TierOneLeverageCapitalToAverageAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Tier One Leverage Capital To Average Assets</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DescriptionOfRegulatoryRequirementsCapitalAdequacyPurposes" xlink:type="locator" xlink:label="us-gaap_DescriptionOfRegulatoryRequirementsCapitalAdequacyPurposes" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DescriptionOfRegulatoryRequirementsCapitalAdequacyPurposes_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Federal banking agencies issued final rules</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DescriptionOfRegulatoryRequirementsCapitalAdequacyPurposes" xlink:to="us-gaap_DescriptionOfRegulatoryRequirementsCapitalAdequacyPurposes_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DescriptionOfRegulatoryRequirementsCapitalAdequacyPurposes_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Description Of Regulatory Requirements Capital Adequacy Purposes</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_CommunityBankLeverageRatioAmount_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Community bank leverage ratio amount</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_CommunityBankLeverageRatioAmount_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Community Bank Leverage Ratio Amount</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_CommonEquityTierOneCapitalToRiskWeightedAssets" xlink:type="locator" xlink:label="pbhc_CommonEquityTierOneCapitalToRiskWeightedAssets" />
    <link:label xml:lang="en-US" xlink:label="pbhc_CommonEquityTierOneCapitalToRiskWeightedAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Common equity tier one capital to risk weighted assets.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_CommonEquityTierOneCapitalToRiskWeightedAssets" xlink:to="pbhc_CommonEquityTierOneCapitalToRiskWeightedAssets_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_CapitalConservationBufferPercentageOfRiskWeightedAssets" xlink:type="locator" xlink:label="pbhc_CapitalConservationBufferPercentageOfRiskWeightedAssets" />
    <link:label xml:lang="en-US" xlink:label="pbhc_CapitalConservationBufferPercentageOfRiskWeightedAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Capital conservation buffer percentage of risk weighted assets.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_CapitalConservationBufferPercentageOfRiskWeightedAssets" xlink:to="pbhc_CapitalConservationBufferPercentageOfRiskWeightedAssets_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_CommonEquityTierOneCapitalToRiskWeightedAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Common equity tier 1 capital to risk-weighted assets</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_CommonEquityTierOneCapitalToRiskWeightedAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Common Equity Tier One Capital To Risk Weighted Assets</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_CapitalConservationBufferPercentageOfRiskWeightedAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Capital conservation buffer percentage of risk weighted assets</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_CapitalConservationBufferPercentageOfRiskWeightedAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Capital Conservation Buffer Percentage Of Risk Weighted Assets</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_TierOneRiskBasedCommonEquity" xlink:type="locator" xlink:label="pbhc_TierOneRiskBasedCommonEquity" />
    <link:label xml:lang="en-US" xlink:label="pbhc_TierOneRiskBasedCommonEquity_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Tier 1 risk based common equity capital as defined in the regulations.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_TierOneRiskBasedCommonEquity" xlink:to="pbhc_TierOneRiskBasedCommonEquity_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ActualAbstract" xlink:type="locator" xlink:label="pbhc_ActualAbstract" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ActualAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Actual Amount [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ActualAbstract" xlink:to="pbhc_ActualAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ActualAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Actual Amount [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ActualAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Actual [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_Capital" xlink:type="locator" xlink:label="us-gaap_Capital" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_Capital_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Total Core Capital (to Risk-Weighted Assets)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_Capital" xlink:to="us-gaap_Capital_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_Capital_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Capital</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_TierOneRiskBasedCapital" xlink:type="locator" xlink:label="us-gaap_TierOneRiskBasedCapital" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TierOneRiskBasedCapital_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Tier 1 Capital (to Risk-Weighted Assets)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_TierOneRiskBasedCapital" xlink:to="us-gaap_TierOneRiskBasedCapital_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TierOneRiskBasedCapital_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Tier One Risk Based Capital</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_TierOneRiskBasedCommonEquity_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Tier 1 Common Equity (to Risk-Weighted Assets)</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_TierOneRiskBasedCommonEquity_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Tier One Risk Based Common Equity</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_TierOneLeverageCapital" xlink:type="locator" xlink:label="us-gaap_TierOneLeverageCapital" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TierOneLeverageCapital_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Tier 1 Capital (to Assets)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_TierOneLeverageCapital" xlink:to="us-gaap_TierOneLeverageCapital_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TierOneLeverageCapital_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Tier One Leverage Capital</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_TierOneRiskBasedCommonEquityToRiskWeightedAssets" xlink:type="locator" xlink:label="pbhc_TierOneRiskBasedCommonEquityToRiskWeightedAssets" />
    <link:label xml:lang="en-US" xlink:label="pbhc_TierOneRiskBasedCommonEquityToRiskWeightedAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The common equity tier 1 capital divided by risk weighted assets as defined in the regulations.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_TierOneRiskBasedCommonEquityToRiskWeightedAssets" xlink:to="pbhc_TierOneRiskBasedCommonEquityToRiskWeightedAssets_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_RiskBasedRatiosAbstract" xlink:type="locator" xlink:label="us-gaap_RiskBasedRatiosAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RiskBasedRatiosAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Actual Ratio [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_RiskBasedRatiosAbstract" xlink:to="us-gaap_RiskBasedRatiosAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RiskBasedRatiosAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Risk Based Ratios [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_CapitalToRiskWeightedAssets" xlink:type="locator" xlink:label="us-gaap_CapitalToRiskWeightedAssets" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CapitalToRiskWeightedAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Total Core Capital (to Risk-Weighted Assets)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_CapitalToRiskWeightedAssets" xlink:to="us-gaap_CapitalToRiskWeightedAssets_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CapitalToRiskWeightedAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Capital To Risk Weighted Assets</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_TierOneRiskBasedCapitalToRiskWeightedAssets" xlink:type="locator" xlink:label="us-gaap_TierOneRiskBasedCapitalToRiskWeightedAssets" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TierOneRiskBasedCapitalToRiskWeightedAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Tier 1 Capital (to Risk-Weighted Assets)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_TierOneRiskBasedCapitalToRiskWeightedAssets" xlink:to="us-gaap_TierOneRiskBasedCapitalToRiskWeightedAssets_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TierOneRiskBasedCapitalToRiskWeightedAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Tier One Risk Based Capital To Risk Weighted Assets</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_TierOneRiskBasedCommonEquityToRiskWeightedAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Tier 1 Common Equity (to Risk-Weighted Assets)</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_TierOneRiskBasedCommonEquityToRiskWeightedAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Tier One Risk Based Common Equity To Risk Weighted Assets</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_TierOneLeverageCapitalToAverageAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Tier 1 Capital (to Assets)</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_TierOneRiskBasedCommonEquityToCapitalRequiredForCapitalAdequacy" xlink:type="locator" xlink:label="pbhc_TierOneRiskBasedCommonEquityToCapitalRequiredForCapitalAdequacy" />
    <link:label xml:lang="en-US" xlink:label="pbhc_TierOneRiskBasedCommonEquityToCapitalRequiredForCapitalAdequacy_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The minimum amount of Tier 1 Risk Based Common Equity required for capital adequacy purposes under the regulatory framework for prompt corrective action.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_TierOneRiskBasedCommonEquityToCapitalRequiredForCapitalAdequacy" xlink:to="pbhc_TierOneRiskBasedCommonEquityToCapitalRequiredForCapitalAdequacy_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ForCapitalAdequacyPurposesAbstract" xlink:type="locator" xlink:label="pbhc_ForCapitalAdequacyPurposesAbstract" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ForCapitalAdequacyPurposesAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Minimum Capital Adequacy Purposes Amount [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ForCapitalAdequacyPurposesAbstract" xlink:to="pbhc_ForCapitalAdequacyPurposesAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ForCapitalAdequacyPurposesAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Minimum Capital Adequacy Purposes Amount [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ForCapitalAdequacyPurposesAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">For Capital Adequacy Purposes [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_CapitalRequiredForCapitalAdequacy" xlink:type="locator" xlink:label="us-gaap_CapitalRequiredForCapitalAdequacy" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CapitalRequiredForCapitalAdequacy_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Total Core Capital (to Risk-Weighted Assets)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_CapitalRequiredForCapitalAdequacy" xlink:to="us-gaap_CapitalRequiredForCapitalAdequacy_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CapitalRequiredForCapitalAdequacy_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Capital Required For Capital Adequacy</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_TierOneRiskBasedCapitalRequiredForCapitalAdequacy" xlink:type="locator" xlink:label="us-gaap_TierOneRiskBasedCapitalRequiredForCapitalAdequacy" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TierOneRiskBasedCapitalRequiredForCapitalAdequacy_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Tier 1 Capital (to Risk-Weighted Assets)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_TierOneRiskBasedCapitalRequiredForCapitalAdequacy" xlink:to="us-gaap_TierOneRiskBasedCapitalRequiredForCapitalAdequacy_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TierOneRiskBasedCapitalRequiredForCapitalAdequacy_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Tier One Risk Based Capital Required For Capital Adequacy</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_TierOneRiskBasedCommonEquityToCapitalRequiredForCapitalAdequacy_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Tier 1 Common Equity (to Risk-Weighted Assets)</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_TierOneRiskBasedCommonEquityToCapitalRequiredForCapitalAdequacy_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Tier One Risk Based Common Equity To Capital Required For Capital Adequacy</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_TierOneLeverageCapitalRequiredForCapitalAdequacy" xlink:type="locator" xlink:label="us-gaap_TierOneLeverageCapitalRequiredForCapitalAdequacy" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TierOneLeverageCapitalRequiredForCapitalAdequacy_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Tier 1 Capital (to Assets)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_TierOneLeverageCapitalRequiredForCapitalAdequacy" xlink:to="us-gaap_TierOneLeverageCapitalRequiredForCapitalAdequacy_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TierOneLeverageCapitalRequiredForCapitalAdequacy_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Tier One Leverage Capital Required For Capital Adequacy</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_TierOneRiskBasedCommonEquityForCapitalAdequacyToRiskWeightedAssets" xlink:type="locator" xlink:label="pbhc_TierOneRiskBasedCommonEquityForCapitalAdequacyToRiskWeightedAssets" />
    <link:label xml:lang="en-US" xlink:label="pbhc_TierOneRiskBasedCommonEquityForCapitalAdequacyToRiskWeightedAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The minimum Tier One Common Equity (common equity tier 1 capital divided by risk-weighted assets) required for capital adequacy purposes under the regulatory framework for prompt corrective action.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_TierOneRiskBasedCommonEquityForCapitalAdequacyToRiskWeightedAssets" xlink:to="pbhc_TierOneRiskBasedCommonEquityForCapitalAdequacyToRiskWeightedAssets_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ForCapitalAdequacyPurposesRatioAbstract" xlink:type="locator" xlink:label="pbhc_ForCapitalAdequacyPurposesRatioAbstract" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ForCapitalAdequacyPurposesRatioAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">For capital adequacy purposes ratio.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ForCapitalAdequacyPurposesRatioAbstract" xlink:to="pbhc_ForCapitalAdequacyPurposesRatioAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ForCapitalAdequacyPurposesRatioAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Minimum Capital Adequacy Purposes Ratio [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ForCapitalAdequacyPurposesRatioAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">For Capital Adequacy Purposes Ratio [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_CapitalRequiredForCapitalAdequacyToRiskWeightedAssets" xlink:type="locator" xlink:label="us-gaap_CapitalRequiredForCapitalAdequacyToRiskWeightedAssets" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CapitalRequiredForCapitalAdequacyToRiskWeightedAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Total Core Capital (to Risk-Weighted Assets)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_CapitalRequiredForCapitalAdequacyToRiskWeightedAssets" xlink:to="us-gaap_CapitalRequiredForCapitalAdequacyToRiskWeightedAssets_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CapitalRequiredForCapitalAdequacyToRiskWeightedAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Capital Required For Capital Adequacy To Risk Weighted Assets</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_TierOneRiskBasedCapitalRequiredForCapitalAdequacyToRiskWeightedAssets" xlink:type="locator" xlink:label="us-gaap_TierOneRiskBasedCapitalRequiredForCapitalAdequacyToRiskWeightedAssets" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TierOneRiskBasedCapitalRequiredForCapitalAdequacyToRiskWeightedAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Tier 1 Capital (to Risk-Weighted Assets)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_TierOneRiskBasedCapitalRequiredForCapitalAdequacyToRiskWeightedAssets" xlink:to="us-gaap_TierOneRiskBasedCapitalRequiredForCapitalAdequacyToRiskWeightedAssets_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TierOneRiskBasedCapitalRequiredForCapitalAdequacyToRiskWeightedAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Tier One Risk Based Capital Required For Capital Adequacy To Risk Weighted Assets</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_TierOneRiskBasedCommonEquityForCapitalAdequacyToRiskWeightedAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Tier 1 Common Equity (to Risk-Weighted Assets)</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_TierOneRiskBasedCommonEquityForCapitalAdequacyToRiskWeightedAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Tier One Risk Based Common Equity For Capital Adequacy To Risk Weighted Assets</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_TierOneLeverageCapitalRequiredForCapitalAdequacyToAverageAssets" xlink:type="locator" xlink:label="us-gaap_TierOneLeverageCapitalRequiredForCapitalAdequacyToAverageAssets" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TierOneLeverageCapitalRequiredForCapitalAdequacyToAverageAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Tier 1 Capital (to Assets)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_TierOneLeverageCapitalRequiredForCapitalAdequacyToAverageAssets" xlink:to="us-gaap_TierOneLeverageCapitalRequiredForCapitalAdequacyToAverageAssets_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TierOneLeverageCapitalRequiredForCapitalAdequacyToAverageAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Tier One Leverage Capital Required For Capital Adequacy To Average Assets</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_TierOneRiskBasedCommonEquitytToBeWellCapitalized" xlink:type="locator" xlink:label="pbhc_TierOneRiskBasedCommonEquitytToBeWellCapitalized" />
    <link:label xml:lang="en-US" xlink:label="pbhc_TierOneRiskBasedCommonEquitytToBeWellCapitalized_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The common equity tier 1 risk-based capital required to be categorized as "well capitalized" under the regulatory framework for prompt corrective action.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_TierOneRiskBasedCommonEquitytToBeWellCapitalized" xlink:to="pbhc_TierOneRiskBasedCommonEquitytToBeWellCapitalized_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ToBeWellCapitalizedUnderPromptCorrectiveActionProvisionsAmountAbstract" xlink:type="locator" xlink:label="pbhc_ToBeWellCapitalizedUnderPromptCorrectiveActionProvisionsAmountAbstract" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ToBeWellCapitalizedUnderPromptCorrectiveActionProvisionsAmountAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">To be well capitalized under prompt corrective action provisions amount.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ToBeWellCapitalizedUnderPromptCorrectiveActionProvisionsAmountAbstract" xlink:to="pbhc_ToBeWellCapitalizedUnderPromptCorrectiveActionProvisionsAmountAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ToBeWellCapitalizedUnderPromptCorrectiveActionProvisionsAmountAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Minimum To Be "Well Capitalized" Under Prompt Corrective Provisions Amount [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ToBeWellCapitalizedUnderPromptCorrectiveActionProvisionsAmountAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">To Be Well Capitalized Under Prompt Corrective Action Provisions Amount [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_CapitalRequiredToBeWellCapitalized" xlink:type="locator" xlink:label="us-gaap_CapitalRequiredToBeWellCapitalized" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CapitalRequiredToBeWellCapitalized_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Total Core Capital (to Risk-Weighted Assets)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_CapitalRequiredToBeWellCapitalized" xlink:to="us-gaap_CapitalRequiredToBeWellCapitalized_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CapitalRequiredToBeWellCapitalized_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Capital Required To Be Well Capitalized</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_TierOneRiskBasedCapitalRequiredToBeWellCapitalized" xlink:type="locator" xlink:label="us-gaap_TierOneRiskBasedCapitalRequiredToBeWellCapitalized" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TierOneRiskBasedCapitalRequiredToBeWellCapitalized_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Tier 1 Capital (to Risk-Weighted Assets)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_TierOneRiskBasedCapitalRequiredToBeWellCapitalized" xlink:to="us-gaap_TierOneRiskBasedCapitalRequiredToBeWellCapitalized_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TierOneRiskBasedCapitalRequiredToBeWellCapitalized_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Tier One Risk Based Capital Required To Be Well Capitalized</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_TierOneRiskBasedCommonEquitytToBeWellCapitalized_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Tier 1 Common Equity (to Risk-Weighted Assets)</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_TierOneRiskBasedCommonEquitytToBeWellCapitalized_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Tier One Risk Based Common Equityt To Be Well Capitalized</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_TierOneLeverageCapitalRequiredToBeWellCapitalized" xlink:type="locator" xlink:label="us-gaap_TierOneLeverageCapitalRequiredToBeWellCapitalized" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TierOneLeverageCapitalRequiredToBeWellCapitalized_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Tier 1 Capital (to Assets)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_TierOneLeverageCapitalRequiredToBeWellCapitalized" xlink:to="us-gaap_TierOneLeverageCapitalRequiredToBeWellCapitalized_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TierOneLeverageCapitalRequiredToBeWellCapitalized_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Tier One Leverage Capital Required To Be Well Capitalized</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_TierOneRiskBasedCommonEquityToBeWellCapitalizedToRiskWeightedAssets" xlink:type="locator" xlink:label="pbhc_TierOneRiskBasedCommonEquityToBeWellCapitalizedToRiskWeightedAssets" />
    <link:label xml:lang="en-US" xlink:label="pbhc_TierOneRiskBasedCommonEquityToBeWellCapitalizedToRiskWeightedAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The common equity tier 1 capital ratio (common equity tier 1 capital divided by risk weighted assets) required to be categorized as "well capitalized" under the regulatory framework for prompt corrective action.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_TierOneRiskBasedCommonEquityToBeWellCapitalizedToRiskWeightedAssets" xlink:to="pbhc_TierOneRiskBasedCommonEquityToBeWellCapitalizedToRiskWeightedAssets_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ToBeWellCapitalizedUnderPromptCorrectiveActionProvisionsRatioAbstract" xlink:type="locator" xlink:label="pbhc_ToBeWellCapitalizedUnderPromptCorrectiveActionProvisionsRatioAbstract" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ToBeWellCapitalizedUnderPromptCorrectiveActionProvisionsRatioAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">To be well capitalized under prompt corrective action provisions ratio.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ToBeWellCapitalizedUnderPromptCorrectiveActionProvisionsRatioAbstract" xlink:to="pbhc_ToBeWellCapitalizedUnderPromptCorrectiveActionProvisionsRatioAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ToBeWellCapitalizedUnderPromptCorrectiveActionProvisionsRatioAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Minimum To Be "Well Capitalized" Under Prompt Corrective Provisions Ratio [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ToBeWellCapitalizedUnderPromptCorrectiveActionProvisionsRatioAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">To Be Well Capitalized Under Prompt Corrective Action Provisions Ratio [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_CapitalRequiredToBeWellCapitalizedToRiskWeightedAssets" xlink:type="locator" xlink:label="us-gaap_CapitalRequiredToBeWellCapitalizedToRiskWeightedAssets" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CapitalRequiredToBeWellCapitalizedToRiskWeightedAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Total Core Capital (to Risk-Weighted Assets)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_CapitalRequiredToBeWellCapitalizedToRiskWeightedAssets" xlink:to="us-gaap_CapitalRequiredToBeWellCapitalizedToRiskWeightedAssets_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CapitalRequiredToBeWellCapitalizedToRiskWeightedAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Capital Required To Be Well Capitalized To Risk Weighted Assets</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_TierOneRiskBasedCapitalRequiredToBeWellCapitalizedToRiskWeightedAssets" xlink:type="locator" xlink:label="us-gaap_TierOneRiskBasedCapitalRequiredToBeWellCapitalizedToRiskWeightedAssets" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TierOneRiskBasedCapitalRequiredToBeWellCapitalizedToRiskWeightedAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Tier 1 Capital (to Risk-Weighted Assets)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_TierOneRiskBasedCapitalRequiredToBeWellCapitalizedToRiskWeightedAssets" xlink:to="us-gaap_TierOneRiskBasedCapitalRequiredToBeWellCapitalizedToRiskWeightedAssets_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TierOneRiskBasedCapitalRequiredToBeWellCapitalizedToRiskWeightedAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Tier One Risk Based Capital Required To Be Well Capitalized To Risk Weighted Assets</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_TierOneRiskBasedCommonEquityToBeWellCapitalizedToRiskWeightedAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Tier 1 Common Equity (to Risk-Weighted Assets)</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_TierOneRiskBasedCommonEquityToBeWellCapitalizedToRiskWeightedAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Tier One Risk Based Common Equity To Be Well Capitalized To Risk Weighted Assets</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_TierOneLeverageCapitalRequiredToBeWellCapitalizedToAverageAssets" xlink:type="locator" xlink:label="us-gaap_TierOneLeverageCapitalRequiredToBeWellCapitalizedToAverageAssets" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TierOneLeverageCapitalRequiredToBeWellCapitalizedToAverageAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Tier 1 Capital (to Assets)</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_TierOneLeverageCapitalRequiredToBeWellCapitalizedToAverageAssets" xlink:to="us-gaap_TierOneLeverageCapitalRequiredToBeWellCapitalizedToAverageAssets_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_TierOneLeverageCapitalRequiredToBeWellCapitalizedToAverageAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Tier One Leverage Capital Required To Be Well Capitalized To Average Assets</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_CapitalRequiredToBeWellCapitalizedWithBuffer" xlink:type="locator" xlink:label="pbhc_CapitalRequiredToBeWellCapitalizedWithBuffer" />
    <link:label xml:lang="en-US" xlink:label="pbhc_CapitalRequiredToBeWellCapitalizedWithBuffer_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The amount of Total Capital required to be categorized as well as capitalized under the regulatory framework for prompt corrective action.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_CapitalRequiredToBeWellCapitalizedWithBuffer" xlink:to="pbhc_CapitalRequiredToBeWellCapitalizedWithBuffer_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_TierOneRiskBasedCapitalRequiredToBeWellCapitalizedWithBuffer" xlink:type="locator" xlink:label="pbhc_TierOneRiskBasedCapitalRequiredToBeWellCapitalizedWithBuffer" />
    <link:label xml:lang="en-US" xlink:label="pbhc_TierOneRiskBasedCapitalRequiredToBeWellCapitalizedWithBuffer_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The amount of Tier 1 Risk Based Capital required to be categorized as well capitalized with buffer under the regulatory framework for prompt corrective action.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_TierOneRiskBasedCapitalRequiredToBeWellCapitalizedWithBuffer" xlink:to="pbhc_TierOneRiskBasedCapitalRequiredToBeWellCapitalizedWithBuffer_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_TierOneRiskBasedCommonEquityToBeWellCapitalizedWithBuffer" xlink:type="locator" xlink:label="pbhc_TierOneRiskBasedCommonEquityToBeWellCapitalizedWithBuffer" />
    <link:label xml:lang="en-US" xlink:label="pbhc_TierOneRiskBasedCommonEquityToBeWellCapitalizedWithBuffer_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Tier one common equity (common equity tier 1 capital divided by risk-weighted assets) required Well-Capitalized with buffer, fully phased.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_TierOneRiskBasedCommonEquityToBeWellCapitalizedWithBuffer" xlink:to="pbhc_TierOneRiskBasedCommonEquityToBeWellCapitalizedWithBuffer_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_TierOneLeverageCapitalRequiredToBeWellCapitalizedWithBuffer" xlink:type="locator" xlink:label="pbhc_TierOneLeverageCapitalRequiredToBeWellCapitalizedWithBuffer" />
    <link:label xml:lang="en-US" xlink:label="pbhc_TierOneLeverageCapitalRequiredToBeWellCapitalizedWithBuffer_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The amount of Tier 1 Leverage Capital required to be categorized as well capitalized with buffer under the regulatory framework for prompt corrective action.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_TierOneLeverageCapitalRequiredToBeWellCapitalizedWithBuffer" xlink:to="pbhc_TierOneLeverageCapitalRequiredToBeWellCapitalizedWithBuffer_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_MinimumForCapitalAdequacyWithBufferAmountAbstract" xlink:type="locator" xlink:label="pbhc_MinimumForCapitalAdequacyWithBufferAmountAbstract" />
    <link:label xml:lang="en-US" xlink:label="pbhc_MinimumForCapitalAdequacyWithBufferAmountAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Minimum for capital adequacy with buffer amount.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_MinimumForCapitalAdequacyWithBufferAmountAbstract" xlink:to="pbhc_MinimumForCapitalAdequacyWithBufferAmountAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_MinimumForCapitalAdequacyWithBufferAmountAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Minimum for Capital Adequacy With Buffer Amount [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MinimumForCapitalAdequacyWithBufferAmountAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Minimum For Capital Adequacy With Buffer Amount [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_CapitalRequiredToBeWellCapitalizedWithBuffer_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Total Core Capital (to Risk-Weighted Assets)</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_CapitalRequiredToBeWellCapitalizedWithBuffer_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Capital Required To Be Well Capitalized With Buffer</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_TierOneRiskBasedCapitalRequiredToBeWellCapitalizedWithBuffer_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Tier 1 Capital (to Risk-Weighted Assets)</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_TierOneRiskBasedCapitalRequiredToBeWellCapitalizedWithBuffer_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Tier One Risk Based Capital Required To Be Well Capitalized With Buffer</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_TierOneRiskBasedCommonEquityToBeWellCapitalizedWithBuffer_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Tier 1 Common Equity (to Risk-Weighted Assets)</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_TierOneRiskBasedCommonEquityToBeWellCapitalizedWithBuffer_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Tier One Risk Based Common Equity To Be Well Capitalized With Buffer</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_TierOneLeverageCapitalRequiredToBeWellCapitalizedWithBuffer_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Tier 1 Capital (to Assets)</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_TierOneLeverageCapitalRequiredToBeWellCapitalizedWithBuffer_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Tier One Leverage Capital Required To Be Well Capitalized With Buffer</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_CapitalRequiredToBeWellCapitalizedWithBufferToRiskWeightedAssets" xlink:type="locator" xlink:label="pbhc_CapitalRequiredToBeWellCapitalizedWithBufferToRiskWeightedAssets" />
    <link:label xml:lang="en-US" xlink:label="pbhc_CapitalRequiredToBeWellCapitalizedWithBufferToRiskWeightedAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The Total risk based capital ratio (total capital divided by risk weighted assets) required to be categorized as "well capitalized with buffer", in the regulatory framework for prompt corrective action.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_CapitalRequiredToBeWellCapitalizedWithBufferToRiskWeightedAssets" xlink:to="pbhc_CapitalRequiredToBeWellCapitalizedWithBufferToRiskWeightedAssets_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_TierOneRiskBasedCapitalRequiredToBeWellCapitalizedWithBufferToRiskWeightedAssets" xlink:type="locator" xlink:label="pbhc_TierOneRiskBasedCapitalRequiredToBeWellCapitalizedWithBufferToRiskWeightedAssets" />
    <link:label xml:lang="en-US" xlink:label="pbhc_TierOneRiskBasedCapitalRequiredToBeWellCapitalizedWithBufferToRiskWeightedAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The Tier 1 capital ratio (Tier 1 capital divided by risk weighted assets) required to be categorized as "well capitalized with buffer" under the regulatory framework for prompt corrective action.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_TierOneRiskBasedCapitalRequiredToBeWellCapitalizedWithBufferToRiskWeightedAssets" xlink:to="pbhc_TierOneRiskBasedCapitalRequiredToBeWellCapitalizedWithBufferToRiskWeightedAssets_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_TierOneRiskBasedCommonEquityToBeWellCapitalizedWithBufferToRiskWeightedAssets" xlink:type="locator" xlink:label="pbhc_TierOneRiskBasedCommonEquityToBeWellCapitalizedWithBufferToRiskWeightedAssets" />
    <link:label xml:lang="en-US" xlink:label="pbhc_TierOneRiskBasedCommonEquityToBeWellCapitalizedWithBufferToRiskWeightedAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The common equity tier 1 capital ratio (common equity tier 1 capital divided by risk weighted assets) required to be categorized as "well capitalized with buffer" under the regulatory framework for prompt corrective action.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_TierOneRiskBasedCommonEquityToBeWellCapitalizedWithBufferToRiskWeightedAssets" xlink:to="pbhc_TierOneRiskBasedCommonEquityToBeWellCapitalizedWithBufferToRiskWeightedAssets_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_TierOneLeverageCapitalRequiredToBeWellCapitalizedWithBufferToAverageAssets" xlink:type="locator" xlink:label="pbhc_TierOneLeverageCapitalRequiredToBeWellCapitalizedWithBufferToAverageAssets" />
    <link:label xml:lang="en-US" xlink:label="pbhc_TierOneLeverageCapitalRequiredToBeWellCapitalizedWithBufferToAverageAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">The amount of Tier 1 Leverage Capital required to be categorized as well capitalized with buffer divided by average assets as defined in the regulatory framework for prompt corrective action.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_TierOneLeverageCapitalRequiredToBeWellCapitalizedWithBufferToAverageAssets" xlink:to="pbhc_TierOneLeverageCapitalRequiredToBeWellCapitalizedWithBufferToAverageAssets_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_MinimumForCapitalAdequacyWithBufferPercentageAbstract" xlink:type="locator" xlink:label="pbhc_MinimumForCapitalAdequacyWithBufferPercentageAbstract" />
    <link:label xml:lang="en-US" xlink:label="pbhc_MinimumForCapitalAdequacyWithBufferPercentageAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Minimum for capital adequacy with buffer percentage.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_MinimumForCapitalAdequacyWithBufferPercentageAbstract" xlink:to="pbhc_MinimumForCapitalAdequacyWithBufferPercentageAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_MinimumForCapitalAdequacyWithBufferPercentageAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Minimum for Capital Adequacy With Buffer Ratio [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MinimumForCapitalAdequacyWithBufferPercentageAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Minimum For Capital Adequacy With Buffer Percentage [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_CapitalRequiredToBeWellCapitalizedWithBufferToRiskWeightedAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Total Core Capital (to Risk-Weighted Assets)</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_CapitalRequiredToBeWellCapitalizedWithBufferToRiskWeightedAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Capital Required To Be Well Capitalized With Buffer To Risk Weighted Assets</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_TierOneRiskBasedCapitalRequiredToBeWellCapitalizedWithBufferToRiskWeightedAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Tier 1 Capital (to Risk-Weighted Assets)</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_TierOneRiskBasedCapitalRequiredToBeWellCapitalizedWithBufferToRiskWeightedAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Tier One Risk Based Capital Required To Be Well Capitalized With Buffer To Risk Weighted Assets</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_TierOneRiskBasedCommonEquityToBeWellCapitalizedWithBufferToRiskWeightedAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Tier 1 Common Equity (to Risk-Weighted Assets)</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_TierOneRiskBasedCommonEquityToBeWellCapitalizedWithBufferToRiskWeightedAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Tier One Risk Based Common Equity To Be Well Capitalized With Buffer To Risk Weighted Assets</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_TierOneLeverageCapitalRequiredToBeWellCapitalizedWithBufferToAverageAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Tier 1 Capital (to Assets)</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_TierOneLeverageCapitalRequiredToBeWellCapitalizedWithBufferToAverageAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Tier One Leverage Capital Required To Be Well Capitalized With Buffer To Average Assets</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_BankingRegulationReserveRequirementRatio" xlink:type="locator" xlink:label="pbhc_BankingRegulationReserveRequirementRatio" />
    <link:label xml:lang="en-US" xlink:label="pbhc_BankingRegulationReserveRequirementRatio_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Banking regulation, reserve requirement ratio.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_BankingRegulationReserveRequirementRatio" xlink:to="pbhc_BankingRegulationReserveRequirementRatio_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_BankingRegulationReserveRequirementRatio_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Federal Reserve Board reserve requirement ratio</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_BankingRegulationReserveRequirementRatio_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Banking Regulation Reserve Requirement Ratio</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DerivativeTable" xlink:type="locator" xlink:label="us-gaap_DerivativeTable" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DerivativeTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Derivative [Table]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DerivativeTable" xlink:to="us-gaap_DerivativeTable_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DerivativeTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Derivative [Table]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_InterestRateSwapMember" xlink:type="locator" xlink:label="us-gaap_InterestRateSwapMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestRateSwapMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Interest Rate Swap [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_InterestRateSwapMember" xlink:to="us-gaap_InterestRateSwapMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestRateSwapMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Interest Rate Swap [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_HedgingDesignationAxis" xlink:type="locator" xlink:label="us-gaap_HedgingDesignationAxis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HedgingDesignationAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Hedging Designation</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_HedgingDesignationAxis" xlink:to="us-gaap_HedgingDesignationAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HedgingDesignationAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Hedging Designation [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_HedgingDesignationDomain" xlink:type="locator" xlink:label="us-gaap_HedgingDesignationDomain" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HedgingDesignationDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Hedging Designation</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_HedgingDesignationDomain" xlink:to="us-gaap_HedgingDesignationDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HedgingDesignationDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Hedging Designation [Domain]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DesignatedAsHedgingInstrumentMember" xlink:type="locator" xlink:label="us-gaap_DesignatedAsHedgingInstrumentMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DesignatedAsHedgingInstrumentMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Designated as Hedging Instrument [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DesignatedAsHedgingInstrumentMember" xlink:to="us-gaap_DesignatedAsHedgingInstrumentMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DesignatedAsHedgingInstrumentMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Designated As Hedging Instrument [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DerivativeInstrumentsGainLossByHedgingRelationshipAxis" xlink:type="locator" xlink:label="us-gaap_DerivativeInstrumentsGainLossByHedgingRelationshipAxis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DerivativeInstrumentsGainLossByHedgingRelationshipAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Hedging Relationship</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DerivativeInstrumentsGainLossByHedgingRelationshipAxis" xlink:to="us-gaap_DerivativeInstrumentsGainLossByHedgingRelationshipAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DerivativeInstrumentsGainLossByHedgingRelationshipAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Derivative Instruments Gain Loss By Hedging Relationship [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_HedgingRelationshipDomain" xlink:type="locator" xlink:label="us-gaap_HedgingRelationshipDomain" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HedgingRelationshipDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Hedging Relationship</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_HedgingRelationshipDomain" xlink:to="us-gaap_HedgingRelationshipDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HedgingRelationshipDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Hedging Relationship [Domain]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FairValueHedgingMember" xlink:type="locator" xlink:label="us-gaap_FairValueHedgingMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueHedgingMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Fair Value Hedges [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FairValueHedgingMember" xlink:to="us-gaap_FairValueHedgingMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueHedgingMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Fair Value Hedging [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_BalanceSheetLocationAxis" xlink:type="locator" xlink:label="us-gaap_BalanceSheetLocationAxis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_BalanceSheetLocationAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Balance Sheet Location</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_BalanceSheetLocationAxis" xlink:to="us-gaap_BalanceSheetLocationAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_BalanceSheetLocationAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Balance Sheet Location [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_BalanceSheetLocationDomain" xlink:type="locator" xlink:label="us-gaap_BalanceSheetLocationDomain" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_BalanceSheetLocationDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Balance Sheet Location</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_BalanceSheetLocationDomain" xlink:to="us-gaap_BalanceSheetLocationDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_BalanceSheetLocationDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Balance Sheet Location [Domain]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_OtherNoncurrentLiabilitiesMember" xlink:type="locator" xlink:label="us-gaap_OtherNoncurrentLiabilitiesMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherNoncurrentLiabilitiesMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Other Liabilities [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_OtherNoncurrentLiabilitiesMember" xlink:to="us-gaap_OtherNoncurrentLiabilitiesMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherNoncurrentLiabilitiesMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Other Noncurrent Liabilities [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_InterestRateCapMember" xlink:type="locator" xlink:label="us-gaap_InterestRateCapMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestRateCapMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Interest Rate Cap [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_InterestRateCapMember" xlink:to="us-gaap_InterestRateCapMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestRateCapMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Interest Rate Cap [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_SwapMember" xlink:type="locator" xlink:label="us-gaap_SwapMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SwapMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Swap [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_SwapMember" xlink:to="us-gaap_SwapMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SwapMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Swap [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DerivativeLineItems" xlink:type="locator" xlink:label="us-gaap_DerivativeLineItems" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DerivativeLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Derivative [Line Items]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DerivativeLineItems" xlink:to="us-gaap_DerivativeLineItems_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DerivativeLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Derivative [Line Items]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DerivativeNotionalAmount" xlink:type="locator" xlink:label="us-gaap_DerivativeNotionalAmount" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DerivativeNotionalAmount_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Notional amount</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DerivativeNotionalAmount" xlink:to="us-gaap_DerivativeNotionalAmount_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DerivativeNotionalAmount_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Derivative Notional Amount</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_LoanReceivableMember" xlink:type="locator" xlink:label="pbhc_LoanReceivableMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_LoanReceivableMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Loan receivable.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_LoanReceivableMember" xlink:to="pbhc_LoanReceivableMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_LoanReceivableMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Loans Receivable [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_LoanReceivableMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Loan Receivable [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AvailableforsaleSecuritiesMember" xlink:type="locator" xlink:label="us-gaap_AvailableforsaleSecuritiesMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AvailableforsaleSecuritiesMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Available-for-sale Securities [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AvailableforsaleSecuritiesMember" xlink:to="us-gaap_AvailableforsaleSecuritiesMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AvailableforsaleSecuritiesMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Availableforsale Securities [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_HedgedAssetFairValueHedge" xlink:type="locator" xlink:label="us-gaap_HedgedAssetFairValueHedge" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HedgedAssetFairValueHedge_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Carrying Amount of the Hedged Assets</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_HedgedAssetFairValueHedge" xlink:to="us-gaap_HedgedAssetFairValueHedge_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HedgedAssetFairValueHedge_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Hedged Asset Fair Value Hedge</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_HedgedAssetFairValueHedgeCumulativeIncreaseDecrease" xlink:type="locator" xlink:label="us-gaap_HedgedAssetFairValueHedgeCumulativeIncreaseDecrease" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HedgedAssetFairValueHedgeCumulativeIncreaseDecrease_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Cumulative Amount of Fair Value Hedging Adjustment Included in The Carrying Amount of the Hedged Assets</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_HedgedAssetFairValueHedgeCumulativeIncreaseDecrease" xlink:to="us-gaap_HedgedAssetFairValueHedgeCumulativeIncreaseDecrease_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HedgedAssetFairValueHedgeCumulativeIncreaseDecrease_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Hedged Asset Fair Value Hedge Cumulative Increase Decrease</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_PrepayableFinancialAssetClosedPortfolioLastOfLayerAmortizedCost" xlink:type="locator" xlink:label="us-gaap_PrepayableFinancialAssetClosedPortfolioLastOfLayerAmortizedCost" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PrepayableFinancialAssetClosedPortfolioLastOfLayerAmortizedCost_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Amortized cost of closed portfolio used in hedging relationship</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_PrepayableFinancialAssetClosedPortfolioLastOfLayerAmortizedCost" xlink:to="us-gaap_PrepayableFinancialAssetClosedPortfolioLastOfLayerAmortizedCost_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PrepayableFinancialAssetClosedPortfolioLastOfLayerAmortizedCost_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Prepayable Financial Asset Closed Portfolio Last Of Layer Amortized Cost</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_HedgedAssetFairValueHedgeCumulativeIncreaseDecrease_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Cumulative basis adjustment associated with hedging relationship</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DerivativeAmountOfHedgedItem" xlink:type="locator" xlink:label="us-gaap_DerivativeAmountOfHedgedItem" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DerivativeAmountOfHedgedItem_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Amount of designated hedging item</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DerivativeAmountOfHedgedItem" xlink:to="us-gaap_DerivativeAmountOfHedgedItem_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DerivativeAmountOfHedgedItem_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Derivative Amount Of Hedged Item</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DerivativeLiabilities" xlink:type="locator" xlink:label="us-gaap_DerivativeLiabilities" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DerivativeLiabilities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Fair value of derivative resulted in net liability position</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DerivativeLiabilities" xlink:to="us-gaap_DerivativeLiabilities_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DerivativeLiabilities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Derivative Liabilities</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DerivativeGainLossOnDerivativeNet" xlink:type="locator" xlink:label="us-gaap_DerivativeGainLossOnDerivativeNet" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DerivativeGainLossOnDerivativeNet_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Loan interest income recognized in earnings</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DerivativeGainLossOnDerivativeNet" xlink:to="us-gaap_DerivativeGainLossOnDerivativeNet_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DerivativeGainLossOnDerivativeNet_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Derivative Gain Loss On Derivative Net</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_DerivativeOneTimePremiumPaidForContract" xlink:type="locator" xlink:label="pbhc_DerivativeOneTimePremiumPaidForContract" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DerivativeOneTimePremiumPaidForContract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Derivative one time premium paid for contract.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_DerivativeOneTimePremiumPaidForContract" xlink:to="pbhc_DerivativeOneTimePremiumPaidForContract_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_DerivativeContractualObligationsToCounterpartyOfContract" xlink:type="locator" xlink:label="pbhc_DerivativeContractualObligationsToCounterpartyOfContract" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DerivativeContractualObligationsToCounterpartyOfContract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Derivative contractual obligations to counterparty of contract.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_DerivativeContractualObligationsToCounterpartyOfContract" xlink:to="pbhc_DerivativeContractualObligationsToCounterpartyOfContract_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_AnnualAverageDerivativeBasisPoints" xlink:type="locator" xlink:label="pbhc_AnnualAverageDerivativeBasisPoints" />
    <link:label xml:lang="en-US" xlink:label="pbhc_AnnualAverageDerivativeBasisPoints_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Annual average derivative basis points.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_AnnualAverageDerivativeBasisPoints" xlink:to="pbhc_AnnualAverageDerivativeBasisPoints_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DerivativeInceptionDates" xlink:type="locator" xlink:label="us-gaap_DerivativeInceptionDates" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DerivativeInceptionDates_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Interest rate cap contract commencement date</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DerivativeInceptionDates" xlink:to="us-gaap_DerivativeInceptionDates_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DerivativeInceptionDates_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Derivative Inception Dates</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DerivativeMaturityDates" xlink:type="locator" xlink:label="us-gaap_DerivativeMaturityDates" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DerivativeMaturityDates_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Interest rate cap contract expiry date</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DerivativeMaturityDates" xlink:to="us-gaap_DerivativeMaturityDates_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DerivativeMaturityDates_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Derivative Maturity Dates</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DerivativeOneTimePremiumPaidForContract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">One time premium paid for contract</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DerivativeOneTimePremiumPaidForContract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Derivative One Time Premium Paid For Contract</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DerivativeContractualObligationsToCounterpartyOfContract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Contractual obligations to counterparty of contract</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DerivativeContractualObligationsToCounterpartyOfContract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Derivative Contractual Obligations To Counterparty Of Contract</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DerivativeTermOfContract" xlink:type="locator" xlink:label="us-gaap_DerivativeTermOfContract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DerivativeTermOfContract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Term of derivative contract</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DerivativeTermOfContract" xlink:to="us-gaap_DerivativeTermOfContract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DerivativeTermOfContract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Derivative Term Of Contract</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AnnualAverageDerivativeBasisPoints_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Basis points relative to notional amount</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AnnualAverageDerivativeBasisPoints_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Annual Average Derivative Basis Points</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DerivativeCapInterestRate" xlink:type="locator" xlink:label="us-gaap_DerivativeCapInterestRate" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DerivativeCapInterestRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Interest rate cap contract strike price</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DerivativeCapInterestRate" xlink:to="us-gaap_DerivativeCapInterestRate_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DerivativeCapInterestRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Derivative Cap Interest Rate</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_DerivativeEffectOnRecordedInterestExpense" xlink:type="locator" xlink:label="pbhc_DerivativeEffectOnRecordedInterestExpense" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DerivativeEffectOnRecordedInterestExpense_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Derivative effect on recorded interest expense.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_DerivativeEffectOnRecordedInterestExpense" xlink:to="pbhc_DerivativeEffectOnRecordedInterestExpense_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DerivativeEffectOnRecordedInterestExpense_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Derivative effect on recorded interest expense</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DerivativeEffectOnRecordedInterestExpense_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Derivative Effect On Recorded Interest Expense</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_DerivativeUnamortizedPremium" xlink:type="locator" xlink:label="pbhc_DerivativeUnamortizedPremium" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DerivativeUnamortizedPremium_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Derivative unamortized premium.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_DerivativeUnamortizedPremium" xlink:to="pbhc_DerivativeUnamortizedPremium_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_DerivativeUnamortizedCap" xlink:type="locator" xlink:label="pbhc_DerivativeUnamortizedCap" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DerivativeUnamortizedCap_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Derivative unamortized cap.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_DerivativeUnamortizedCap" xlink:to="pbhc_DerivativeUnamortizedCap_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_CashFlowHedgingMember" xlink:type="locator" xlink:label="us-gaap_CashFlowHedgingMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CashFlowHedgingMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Cash Flow Hedging [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_CashFlowHedgingMember" xlink:to="us-gaap_CashFlowHedgingMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CashFlowHedgingMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Cash Flow Hedging [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DerivativeUnamortizedPremium_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Total unamortized premium</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DerivativeUnamortizedPremium_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Derivative Unamortized Premium</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DerivativeLossOnDerivative" xlink:type="locator" xlink:label="us-gaap_DerivativeLossOnDerivative" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DerivativeLossOnDerivative_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Total loss in comprehensive income</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DerivativeLossOnDerivative" xlink:to="us-gaap_DerivativeLossOnDerivative_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DerivativeLossOnDerivative_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Derivative Loss On Derivative</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DerivativeUnamortizedCap_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Total unamortized cap</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DerivativeUnamortizedCap_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Derivative Unamortized Cap</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_EscrowDeposit" xlink:type="locator" xlink:label="us-gaap_EscrowDeposit" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EscrowDeposit_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedPeriodStartLabel">Balance as of December 31:</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_EscrowDeposit" xlink:to="us-gaap_EscrowDeposit_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EscrowDeposit_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Escrow Deposit</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_DerivativeGainLossOnDerivativeNet_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Amount of gains (losses) recognized in other comprehensive income</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherComprehensiveIncomeLossCashFlowHedgeGainLossAfterReclassificationBeforeTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Losses in other comprehensive income:</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_NumberOfSwapContracts" xlink:type="locator" xlink:label="pbhc_NumberOfSwapContracts" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfSwapContracts_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Number of swap contracts.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_NumberOfSwapContracts" xlink:to="pbhc_NumberOfSwapContracts_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_NumberOfHedgeSecurities" xlink:type="locator" xlink:label="pbhc_NumberOfHedgeSecurities" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfHedgeSecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Number of hedge securities.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_NumberOfHedgeSecurities" xlink:to="pbhc_NumberOfHedgeSecurities_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfHedgeSecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Number of hedge securities</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfHedgeSecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Number Of Hedge Securities</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfSwapContracts_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Number of swap contracts</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfSwapContracts_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Number Of Swap Contracts</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DerivativeAssets" xlink:type="locator" xlink:label="us-gaap_DerivativeAssets" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DerivativeAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Fair value of derivative resulted in net asset position</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DerivativeAssets" xlink:to="us-gaap_DerivativeAssets_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DerivativeAssets_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Derivative Assets</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_DerivativeContractCashInEscrowUnderCollateralArrangements" xlink:type="locator" xlink:label="pbhc_DerivativeContractCashInEscrowUnderCollateralArrangements" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DerivativeContractCashInEscrowUnderCollateralArrangements_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Derivative contract cash in escrow under collateral arrangements.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_DerivativeContractCashInEscrowUnderCollateralArrangements" xlink:to="pbhc_DerivativeContractCashInEscrowUnderCollateralArrangements_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DerivativeContractCashInEscrowUnderCollateralArrangements_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Cash in escrow under collateral arrangements</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DerivativeContractCashInEscrowUnderCollateralArrangements_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Derivative Contract Cash In Escrow Under Collateral Arrangements</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DebtSecuritiesAvailableForSaleTable" xlink:type="locator" xlink:label="us-gaap_DebtSecuritiesAvailableForSaleTable" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtSecuritiesAvailableForSaleTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Debt Securities Available For Sale [Table]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DebtSecuritiesAvailableForSaleTable" xlink:to="us-gaap_DebtSecuritiesAvailableForSaleTable_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtSecuritiesAvailableForSaleTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Debt Securities Available For Sale [Table]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_InvestmentTypeAxis" xlink:type="locator" xlink:label="us-gaap_InvestmentTypeAxis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InvestmentTypeAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Investment Type</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_InvestmentTypeAxis" xlink:to="us-gaap_InvestmentTypeAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InvestmentTypeAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Investment Type [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_InvestmentTypeCategorizationMember" xlink:type="locator" xlink:label="us-gaap_InvestmentTypeCategorizationMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InvestmentTypeCategorizationMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Investments</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_InvestmentTypeCategorizationMember" xlink:to="us-gaap_InvestmentTypeCategorizationMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InvestmentTypeCategorizationMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Investment Type Categorization [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_EquitySecuritiesInvestmentSummaryMember" xlink:type="locator" xlink:label="us-gaap_EquitySecuritiesInvestmentSummaryMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EquitySecuritiesInvestmentSummaryMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Equity Investment [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_EquitySecuritiesInvestmentSummaryMember" xlink:to="us-gaap_EquitySecuritiesInvestmentSummaryMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EquitySecuritiesInvestmentSummaryMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Equity Securities Investment Summary [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_EquitySecurityInvestmentsOneMember" xlink:type="locator" xlink:label="pbhc_EquitySecurityInvestmentsOneMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_EquitySecurityInvestmentsOneMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Equity security investments one.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_EquitySecurityInvestmentsOneMember" xlink:to="pbhc_EquitySecurityInvestmentsOneMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_EquitySecurityInvestmentsOneMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Equity Security Investment 1 [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_EquitySecurityInvestmentsOneMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Equity Security Investments One [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_EquitySecurityInvestmentsTwoMember" xlink:type="locator" xlink:label="pbhc_EquitySecurityInvestmentsTwoMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_EquitySecurityInvestmentsTwoMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Equity security investments two.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_EquitySecurityInvestmentsTwoMember" xlink:to="pbhc_EquitySecurityInvestmentsTwoMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_EquitySecurityInvestmentsTwoMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Equity Security Investment 2 [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_EquitySecurityInvestmentsTwoMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Equity Security Investments Two [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ScheduleOfAvailableForSaleSecuritiesLineItems" xlink:type="locator" xlink:label="us-gaap_ScheduleOfAvailableForSaleSecuritiesLineItems" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfAvailableForSaleSecuritiesLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Schedule Of Available For Sale Securities [Line Items]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ScheduleOfAvailableForSaleSecuritiesLineItems" xlink:to="us-gaap_ScheduleOfAvailableForSaleSecuritiesLineItems_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfAvailableForSaleSecuritiesLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Available For Sale Securities [Line Items]</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPosition12MonthsOrLonger_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/positiveVerboseLabel">Available for sale securities aggregate market value</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_DebtInvestmentSecuritiesAbstract" xlink:type="locator" xlink:label="pbhc_DebtInvestmentSecuritiesAbstract" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DebtInvestmentSecuritiesAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Debt investment securities [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_DebtInvestmentSecuritiesAbstract" xlink:to="pbhc_DebtInvestmentSecuritiesAbstract_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_EquityInvestmentSecuritiesAbstract" xlink:type="locator" xlink:label="pbhc_EquityInvestmentSecuritiesAbstract" />
    <link:label xml:lang="en-US" xlink:label="pbhc_EquityInvestmentSecuritiesAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Equity investment securities [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_EquityInvestmentSecuritiesAbstract" xlink:to="pbhc_EquityInvestmentSecuritiesAbstract_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisTable" xlink:type="locator" xlink:label="us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisTable" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Fair Value Assets And Liabilities Measured On Recurring And Nonrecurring Basis [Table]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisTable" xlink:to="us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisTable_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Fair Value Assets And Liabilities Measured On Recurring And Nonrecurring Basis [Table]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FairValueByMeasurementFrequencyAxis" xlink:type="locator" xlink:label="us-gaap_FairValueByMeasurementFrequencyAxis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueByMeasurementFrequencyAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Measurement Frequency</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FairValueByMeasurementFrequencyAxis" xlink:to="us-gaap_FairValueByMeasurementFrequencyAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueByMeasurementFrequencyAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Fair Value By Measurement Frequency [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FairValueMeasurementFrequencyDomain" xlink:type="locator" xlink:label="us-gaap_FairValueMeasurementFrequencyDomain" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueMeasurementFrequencyDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Measurement Frequency</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FairValueMeasurementFrequencyDomain" xlink:to="us-gaap_FairValueMeasurementFrequencyDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueMeasurementFrequencyDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Fair Value Measurement Frequency [Domain]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FairValueMeasurementsRecurringMember" xlink:type="locator" xlink:label="us-gaap_FairValueMeasurementsRecurringMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueMeasurementsRecurringMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Recurring Basis [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FairValueMeasurementsRecurringMember" xlink:to="us-gaap_FairValueMeasurementsRecurringMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueMeasurementsRecurringMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Fair Value Measurements Recurring [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FairValueInputsLevel12And3Member" xlink:type="locator" xlink:label="us-gaap_FairValueInputsLevel12And3Member" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueInputsLevel12And3Member_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Level 1, 2 and 3 [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FairValueInputsLevel12And3Member" xlink:to="us-gaap_FairValueInputsLevel12And3Member_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueInputsLevel12And3Member_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Fair Value Inputs Level12 And3 [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FairValueMeasuredAtNetAssetValuePerShareMember" xlink:type="locator" xlink:label="us-gaap_FairValueMeasuredAtNetAssetValuePerShareMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueMeasuredAtNetAssetValuePerShareMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Net Asset Value [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FairValueMeasuredAtNetAssetValuePerShareMember" xlink:to="us-gaap_FairValueMeasuredAtNetAssetValuePerShareMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueMeasuredAtNetAssetValuePerShareMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Fair Value Measured At Net Asset Value Per Share [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FairValueByMeasurementBasisAxis" xlink:type="locator" xlink:label="us-gaap_FairValueByMeasurementBasisAxis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueByMeasurementBasisAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Measurement Basis</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FairValueByMeasurementBasisAxis" xlink:to="us-gaap_FairValueByMeasurementBasisAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueByMeasurementBasisAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Fair Value By Measurement Basis [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_PortionAtFairValueFairValueDisclosureMember" xlink:type="locator" xlink:label="us-gaap_PortionAtFairValueFairValueDisclosureMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PortionAtFairValueFairValueDisclosureMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Fair Value Disclosure Item Amounts [Default]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_PortionAtFairValueFairValueDisclosureMember" xlink:to="us-gaap_PortionAtFairValueFairValueDisclosureMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PortionAtFairValueFairValueDisclosureMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Portion At Fair Value Fair Value Disclosure [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FairValueDisclosureItemAmountsDomain" xlink:type="locator" xlink:label="us-gaap_FairValueDisclosureItemAmountsDomain" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueDisclosureItemAmountsDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Fair Value Measurement</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FairValueDisclosureItemAmountsDomain" xlink:to="us-gaap_FairValueDisclosureItemAmountsDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueDisclosureItemAmountsDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Fair Value Disclosure Item Amounts [Domain]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_EstimateOfFairValueFairValueDisclosureMember" xlink:type="locator" xlink:label="us-gaap_EstimateOfFairValueFairValueDisclosureMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EstimateOfFairValueFairValueDisclosureMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Total Fair Value [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_EstimateOfFairValueFairValueDisclosureMember" xlink:to="us-gaap_EstimateOfFairValueFairValueDisclosureMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EstimateOfFairValueFairValueDisclosureMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Estimate Of Fair Value Fair Value Disclosure [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisLineItems" xlink:type="locator" xlink:label="us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisLineItems" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Fair Value Assets And Liabilities Measured On Recurring And Nonrecurring Basis [Line Items]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisLineItems" xlink:to="us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisLineItems_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Fair Value Assets And Liabilities Measured On Recurring And Nonrecurring Basis [Line Items]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DebtInvestmentSecuritiesAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Debt investment securities:</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DebtInvestmentSecuritiesAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Debt Investment Securities [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_UsTreasuryAgenciesAndGsesFairValueDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">US Treasury, agencies and GSEs</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_UsTreasuryAgenciesAndGsesFairValueDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Us Treasury Agencies And Gses Fair Value Disclosure</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_StateAndPoliticalSubdivisionsFairValueDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">State and political subdivisions</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_StateAndPoliticalSubdivisionsFairValueDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">State And Political Subdivisions Fair Value Disclosure</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_CorporateFairValueDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Corporate</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_CorporateFairValueDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Corporate Fair Value Disclosure</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AssetBackedSecuritiesDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Asset backed securities</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AssetBackedSecuritiesDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Asset Backed Securities Disclosure</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ResidentialMortgageBackedAgencyFairValueDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Residential mortgage-backed - US agency</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ResidentialMortgageBackedAgencyFairValueDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Residential Mortgage Backed Agency Fair Value Disclosure</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_CollateralizedMortgageObligationsAgencyFairValueDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Collateralized mortgage obligations - US agency</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_CollateralizedMortgageObligationsAgencyFairValueDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Collateralized Mortgage Obligations Agency Fair Value Disclosure</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_CollateralizedMortgageObligationsPrivateLabelFairValueDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Collateralized mortgage obligations - Private label</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_CollateralizedMortgageObligationsPrivateLabelFairValueDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Collateralized Mortgage Obligations Private Label Fair Value Disclosure</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AvailableForSaleDebtAndEquitySecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Investment securities - available-for-sale</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_InvestmentsFairValueDisclosure" xlink:type="locator" xlink:label="us-gaap_InvestmentsFairValueDisclosure" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InvestmentsFairValueDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Total available-for-sale securities</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_InvestmentsFairValueDisclosure" xlink:to="us-gaap_InvestmentsFairValueDisclosure_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InvestmentsFairValueDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Investments Fair Value Disclosure</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_EquityInvestmentSecuritiesAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Equity investment securities:</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_EquityInvestmentSecuritiesAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Equity Investment Securities [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_EquitySecuritiesFvNi_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/positiveTerseLabel">Marketable equity securities</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_InterestRateDerivativesAtFairValueNetAbstract" xlink:type="locator" xlink:label="us-gaap_InterestRateDerivativesAtFairValueNetAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestRateDerivativesAtFairValueNetAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Interest rate derivatives, at fair value, net :</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_InterestRateDerivativesAtFairValueNetAbstract" xlink:to="us-gaap_InterestRateDerivativesAtFairValueNetAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestRateDerivativesAtFairValueNetAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Interest Rate Derivatives At Fair Value Net [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_InterestRateFairValueHedgeDerivativeAtFairValueNet" xlink:type="locator" xlink:label="us-gaap_InterestRateFairValueHedgeDerivativeAtFairValueNet" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestRateFairValueHedgeDerivativeAtFairValueNet_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Interest rate swap derivative fair value hedge</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_InterestRateFairValueHedgeDerivativeAtFairValueNet" xlink:to="us-gaap_InterestRateFairValueHedgeDerivativeAtFairValueNet_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestRateFairValueHedgeDerivativeAtFairValueNet_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Interest Rate Fair Value Hedge Derivative At Fair Value Net</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_InterestRateCashFlowHedgeDerivativeAtFairValueNet" xlink:type="locator" xlink:label="us-gaap_InterestRateCashFlowHedgeDerivativeAtFairValueNet" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestRateCashFlowHedgeDerivativeAtFairValueNet_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Interest rate swap derivative cash flow hedge</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_InterestRateCashFlowHedgeDerivativeAtFairValueNet" xlink:to="us-gaap_InterestRateCashFlowHedgeDerivativeAtFairValueNet_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestRateCashFlowHedgeDerivativeAtFairValueNet_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Interest Rate Cash Flow Hedge Derivative At Fair Value Net</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_NonrecurringBasisAbstract" xlink:type="locator" xlink:label="pbhc_NonrecurringBasisAbstract" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NonrecurringBasisAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Nonrecurring basis [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_NonrecurringBasisAbstract" xlink:to="pbhc_NonrecurringBasisAbstract_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ImpairedLoansFairValueDisclosure" xlink:type="locator" xlink:label="pbhc_ImpairedLoansFairValueDisclosure" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ImpairedLoansFairValueDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">This element represents the portion of the balance sheet assertion valued at fair value by the entity whether such amount is presented as a separate caption or as a parenthetical disclosure. Additionally, this element may be used in connection with the fair value disclosures required in the footnote disclosures to the financial statements. The element may be used in both the balance sheet and disclosure in the same submission. This item represents impaired loans as presented on the statement of financial position which may include mortgage loans, finance receivables, or any other impaired loans which are due to the Company as of the balance sheet date.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ImpairedLoansFairValueDisclosure" xlink:to="pbhc_ImpairedLoansFairValueDisclosure_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FairValueMeasurementsNonrecurringMember" xlink:type="locator" xlink:label="us-gaap_FairValueMeasurementsNonrecurringMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueMeasurementsNonrecurringMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Nonrecurring Basis [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FairValueMeasurementsNonrecurringMember" xlink:to="us-gaap_FairValueMeasurementsNonrecurringMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueMeasurementsNonrecurringMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Fair Value Measurements Nonrecurring [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NonrecurringBasisAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Nonrecurring basis [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NonrecurringBasisAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Nonrecurring Basis [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ImpairedLoansFairValueDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Impaired loans</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ImpairedLoansFairValueDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Impaired Loans Fair Value Disclosure</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisValuationTechniquesTable" xlink:type="locator" xlink:label="us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisValuationTechniquesTable" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisValuationTechniquesTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Fair Value Assets And Liabilities Measured On Recurring And Nonrecurring Basis Valuation Techniques [Table]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisValuationTechniquesTable" xlink:to="us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisValuationTechniquesTable_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisValuationTechniquesTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Fair Value Assets And Liabilities Measured On Recurring And Nonrecurring Basis Valuation Techniques [Table]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FairValueByAssetClassAxis" xlink:type="locator" xlink:label="us-gaap_FairValueByAssetClassAxis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueByAssetClassAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Asset Class</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FairValueByAssetClassAxis" xlink:to="us-gaap_FairValueByAssetClassAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueByAssetClassAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Fair Value By Asset Class [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationByAssetClassDomain" xlink:type="locator" xlink:label="us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationByAssetClassDomain" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationByAssetClassDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Asset Class</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationByAssetClassDomain" xlink:to="us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationByAssetClassDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationByAssetClassDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Fair Value Assets Measured On Recurring Basis Unobservable Input Reconciliation By Asset Class [Domain]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ImpairedLoansMember" xlink:type="locator" xlink:label="pbhc_ImpairedLoansMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ImpairedLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Loans classified as impaired.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ImpairedLoansMember" xlink:to="pbhc_ImpairedLoansMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ImpairedLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Impaired Loans [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ImpairedLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Impaired Loans [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/srt/2020/elts/srt-2020-01-31.xsd#srt_WeightedAverageMember" xlink:type="locator" xlink:label="srt_WeightedAverageMember" />
    <link:label xml:lang="en-US" xlink:label="srt_WeightedAverageMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Weighted Average [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="srt_WeightedAverageMember" xlink:to="srt_WeightedAverageMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="srt_WeightedAverageMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Weighted Average [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_MeasurementInputTypeAxis" xlink:type="locator" xlink:label="us-gaap_MeasurementInputTypeAxis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_MeasurementInputTypeAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Measurement Input Type</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_MeasurementInputTypeAxis" xlink:to="us-gaap_MeasurementInputTypeAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_MeasurementInputTypeAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Measurement Input Type [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_MeasurementInputTypeDomain" xlink:type="locator" xlink:label="us-gaap_MeasurementInputTypeDomain" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_MeasurementInputTypeDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Measurement Input Type</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_MeasurementInputTypeDomain" xlink:to="us-gaap_MeasurementInputTypeDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_MeasurementInputTypeDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Measurement Input Type [Domain]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_MeasurementInputDiscountRateMember" xlink:type="locator" xlink:label="us-gaap_MeasurementInputDiscountRateMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_MeasurementInputDiscountRateMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Measurement Input, Discount Rate [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_MeasurementInputDiscountRateMember" xlink:to="us-gaap_MeasurementInputDiscountRateMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_MeasurementInputDiscountRateMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Measurement Input Discount Rate [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ValuationTechniqueAxis" xlink:type="locator" xlink:label="us-gaap_ValuationTechniqueAxis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ValuationTechniqueAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Valuation Approach and Technique</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ValuationTechniqueAxis" xlink:to="us-gaap_ValuationTechniqueAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ValuationTechniqueAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Valuation Technique [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ValuationTechniqueDomain" xlink:type="locator" xlink:label="us-gaap_ValuationTechniqueDomain" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ValuationTechniqueDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Valuation Approach and Technique</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ValuationTechniqueDomain" xlink:to="us-gaap_ValuationTechniqueDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ValuationTechniqueDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Valuation Technique [Domain]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_MarketApproachValuationTechniqueMember" xlink:type="locator" xlink:label="us-gaap_MarketApproachValuationTechniqueMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_MarketApproachValuationTechniqueMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Appraisal of Collateral - Appraisal Adjustments [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_MarketApproachValuationTechniqueMember" xlink:to="us-gaap_MarketApproachValuationTechniqueMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_MarketApproachValuationTechniqueMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Market Approach Valuation Technique [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_CostApproachValuationTechniqueMember" xlink:type="locator" xlink:label="us-gaap_CostApproachValuationTechniqueMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CostApproachValuationTechniqueMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Appraisal of Collateral - Cost to Sell [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_CostApproachValuationTechniqueMember" xlink:to="us-gaap_CostApproachValuationTechniqueMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CostApproachValuationTechniqueMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Cost Approach Valuation Technique [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisValuationTechniquesLineItems" xlink:type="locator" xlink:label="us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisValuationTechniquesLineItems" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisValuationTechniquesLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Fair Value Assets And Liabilities Measured On Recurring And Nonrecurring Basis Valuation Techniques [Line Items]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisValuationTechniquesLineItems" xlink:to="us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisValuationTechniquesLineItems_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisValuationTechniquesLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Fair Value Assets And Liabilities Measured On Recurring And Nonrecurring Basis Valuation Techniques [Line Items]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AlternativeInvestmentMeasurementInput" xlink:type="locator" xlink:label="us-gaap_AlternativeInvestmentMeasurementInput" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AlternativeInvestmentMeasurementInput_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Fair value inputs, discount rate</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AlternativeInvestmentMeasurementInput" xlink:to="us-gaap_AlternativeInvestmentMeasurementInput_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AlternativeInvestmentMeasurementInput_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Alternative Investment Measurement Input</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_InterestReceivableFairValueDisclosure" xlink:type="locator" xlink:label="pbhc_InterestReceivableFairValueDisclosure" />
    <link:label xml:lang="en-US" xlink:label="pbhc_InterestReceivableFairValueDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">This element represents the portion of the balance sheet assertion valued at fair value of accrued interest receivables.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_InterestReceivableFairValueDisclosure" xlink:to="pbhc_InterestReceivableFairValueDisclosure_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_TimeDepositsFairValueDisclosure" xlink:type="locator" xlink:label="pbhc_TimeDepositsFairValueDisclosure" />
    <link:label xml:lang="en-US" xlink:label="pbhc_TimeDepositsFairValueDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Fair value disclosure of certificate of deposit or savings account held by a bank or other financial institution for a short-term specified period of time. Because of their short-term, time deposits are considered highly liquid investments that are both readily convertible to known amounts of cash and so near their maturity that they present insignificant risk of changes in value because of changes in interest rates. Generally, only investments with original maturities of three months or less qualify under that definition</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_TimeDepositsFairValueDisclosure" xlink:to="pbhc_TimeDepositsFairValueDisclosure_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_JuniorSubordinatedDebenturesFairValueDisclosure" xlink:type="locator" xlink:label="pbhc_JuniorSubordinatedDebenturesFairValueDisclosure" />
    <link:label xml:lang="en-US" xlink:label="pbhc_JuniorSubordinatedDebenturesFairValueDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">This element represents the portion of the balance sheet assertion valued at fair value by the entity whether such amount is presented as a separate caption or as a parenthetical disclosure. Additionally, this element may be used in connection with the fair value disclosures required in the footnote disclosures to the financial statements. The element may be used in both the balance sheet and disclosure. This item represents the aggregate of all Junior subordinated debentures.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_JuniorSubordinatedDebenturesFairValueDisclosure" xlink:to="pbhc_JuniorSubordinatedDebenturesFairValueDisclosure_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_AccruedInterestPayableFairValueDisclosure" xlink:type="locator" xlink:label="pbhc_AccruedInterestPayableFairValueDisclosure" />
    <link:label xml:lang="en-US" xlink:label="pbhc_AccruedInterestPayableFairValueDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">This element represents the portion of the balance sheet assertion valued at fair value by the entity whether such amount is presented as a separate caption or as a parenthetical disclosure. Additionally, this element may be used in connection with the fair value disclosures required in the footnote disclosures to the financial statements. The element may be used in both the balance sheet and disclosure. This item represents the aggregate of all accrued interest payable.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_AccruedInterestPayableFairValueDisclosure" xlink:to="pbhc_AccruedInterestPayableFairValueDisclosure_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FairValueByBalanceSheetGroupingTable" xlink:type="locator" xlink:label="us-gaap_FairValueByBalanceSheetGroupingTable" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueByBalanceSheetGroupingTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Fair Value By Balance Sheet Grouping [Table]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FairValueByBalanceSheetGroupingTable" xlink:to="us-gaap_FairValueByBalanceSheetGroupingTable_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueByBalanceSheetGroupingTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Fair Value By Balance Sheet Grouping [Table]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_CarryingReportedAmountFairValueDisclosureMember" xlink:type="locator" xlink:label="us-gaap_CarryingReportedAmountFairValueDisclosureMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CarryingReportedAmountFairValueDisclosureMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Carrying Amounts [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_CarryingReportedAmountFairValueDisclosureMember" xlink:to="us-gaap_CarryingReportedAmountFairValueDisclosureMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CarryingReportedAmountFairValueDisclosureMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Carrying Reported Amount Fair Value Disclosure [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_EstimateOfFairValueFairValueDisclosureMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Estimated Fair Values [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FairValueBalanceSheetGroupingFinancialStatementCaptionsLineItems" xlink:type="locator" xlink:label="us-gaap_FairValueBalanceSheetGroupingFinancialStatementCaptionsLineItems" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueBalanceSheetGroupingFinancialStatementCaptionsLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Fair Value Balance Sheet Grouping Financial Statement Captions [Line Items]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FairValueBalanceSheetGroupingFinancialStatementCaptionsLineItems" xlink:to="us-gaap_FairValueBalanceSheetGroupingFinancialStatementCaptionsLineItems_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FairValueBalanceSheetGroupingFinancialStatementCaptionsLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Fair Value Balance Sheet Grouping Financial Statement Captions [Line Items]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FinancialInstrumentsFinancialAssetsBalanceSheetGroupingsAbstract" xlink:type="locator" xlink:label="us-gaap_FinancialInstrumentsFinancialAssetsBalanceSheetGroupingsAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancialInstrumentsFinancialAssetsBalanceSheetGroupingsAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Financial assets:</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FinancialInstrumentsFinancialAssetsBalanceSheetGroupingsAbstract" xlink:to="us-gaap_FinancialInstrumentsFinancialAssetsBalanceSheetGroupingsAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancialInstrumentsFinancialAssetsBalanceSheetGroupingsAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Financial Instruments Financial Assets Balance Sheet Groupings [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_CashAndCashEquivalentsFairValueDisclosure" xlink:type="locator" xlink:label="us-gaap_CashAndCashEquivalentsFairValueDisclosure" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CashAndCashEquivalentsFairValueDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Cash and cash equivalents</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_CashAndCashEquivalentsFairValueDisclosure" xlink:to="us-gaap_CashAndCashEquivalentsFairValueDisclosure_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CashAndCashEquivalentsFairValueDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Cash And Cash Equivalents Fair Value Disclosure</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AvailableForSaleDebtAndEquitySecurities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/positiveLabel">Investment securities - available-for-sale</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_EquitySecuritiesFvNi_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/positiveLabel">Investment securities - marketable equity</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_HeldToMaturitySecuritiesFairValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/positiveLabel">Investment securities - held-to-maturity</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_InvestmentInFederalHomeLoanBankStockFairValueDisclosure" xlink:type="locator" xlink:label="us-gaap_InvestmentInFederalHomeLoanBankStockFairValueDisclosure" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InvestmentInFederalHomeLoanBankStockFairValueDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Federal Home Loan Bank stock</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_InvestmentInFederalHomeLoanBankStockFairValueDisclosure" xlink:to="us-gaap_InvestmentInFederalHomeLoanBankStockFairValueDisclosure_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InvestmentInFederalHomeLoanBankStockFairValueDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Investment In Federal Home Loan Bank Stock Fair Value Disclosure</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LoansHeldForSaleFairValueDisclosure" xlink:type="locator" xlink:label="us-gaap_LoansHeldForSaleFairValueDisclosure" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LoansHeldForSaleFairValueDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Net loans</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LoansHeldForSaleFairValueDisclosure" xlink:to="us-gaap_LoansHeldForSaleFairValueDisclosure_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LoansHeldForSaleFairValueDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Loans Held For Sale Fair Value Disclosure</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_InterestReceivableFairValueDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Accrued interest receivable</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_InterestReceivableFairValueDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Interest Receivable Fair Value Disclosure</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FinancialInstrumentsFinancialLiabilitiesBalanceSheetGroupingsAbstract" xlink:type="locator" xlink:label="us-gaap_FinancialInstrumentsFinancialLiabilitiesBalanceSheetGroupingsAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancialInstrumentsFinancialLiabilitiesBalanceSheetGroupingsAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Financial liabilities:</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FinancialInstrumentsFinancialLiabilitiesBalanceSheetGroupingsAbstract" xlink:to="us-gaap_FinancialInstrumentsFinancialLiabilitiesBalanceSheetGroupingsAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinancialInstrumentsFinancialLiabilitiesBalanceSheetGroupingsAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Financial Instruments Financial Liabilities Balance Sheet Groupings [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DepositsFairValueDisclosure" xlink:type="locator" xlink:label="us-gaap_DepositsFairValueDisclosure" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DepositsFairValueDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Demand Deposits, Savings, NOW and MMDA</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DepositsFairValueDisclosure" xlink:to="us-gaap_DepositsFairValueDisclosure_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DepositsFairValueDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deposits Fair Value Disclosure</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_TimeDepositsFairValueDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Time Deposits</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_TimeDepositsFairValueDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Time Deposits Fair Value Disclosure</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DebtInstrumentFairValue" xlink:type="locator" xlink:label="us-gaap_DebtInstrumentFairValue" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtInstrumentFairValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Borrowings</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DebtInstrumentFairValue" xlink:to="us-gaap_DebtInstrumentFairValue_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtInstrumentFairValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Debt Instrument Fair Value</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_JuniorSubordinatedDebenturesFairValueDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Subordinated loans</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_JuniorSubordinatedDebenturesFairValueDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Junior Subordinated Debentures Fair Value Disclosure</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AccruedInterestPayableFairValueDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Accrued interest payable</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AccruedInterestPayableFairValueDisclosure_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Accrued Interest Payable Fair Value Disclosure</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_InterestRateFairValueHedgeLiabilityAtFairValue" xlink:type="locator" xlink:label="us-gaap_InterestRateFairValueHedgeLiabilityAtFairValue" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestRateFairValueHedgeLiabilityAtFairValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Interest rate swap derivative fair value hedges</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_InterestRateFairValueHedgeLiabilityAtFairValue" xlink:to="us-gaap_InterestRateFairValueHedgeLiabilityAtFairValue_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestRateFairValueHedgeLiabilityAtFairValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Interest Rate Fair Value Hedge Liability At Fair Value</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_InterestRateCashFlowHedgeLiabilityAtFairValue" xlink:type="locator" xlink:label="us-gaap_InterestRateCashFlowHedgeLiabilityAtFairValue" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestRateCashFlowHedgeLiabilityAtFairValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Interest rate swap derivative cash flow hedges</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_InterestRateCashFlowHedgeLiabilityAtFairValue" xlink:to="us-gaap_InterestRateCashFlowHedgeLiabilityAtFairValue_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestRateCashFlowHedgeLiabilityAtFairValue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Interest Rate Cash Flow Hedge Liability At Fair Value</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/srt/2020/elts/srt-2020-01-31.xsd#srt_ScheduleOfCondensedFinancialStatementsTable" xlink:type="locator" xlink:label="srt_ScheduleOfCondensedFinancialStatementsTable" />
    <link:label xml:lang="en-US" xlink:label="srt_ScheduleOfCondensedFinancialStatementsTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Schedule Of Condensed Financial Statements [Table]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="srt_ScheduleOfCondensedFinancialStatementsTable" xlink:to="srt_ScheduleOfCondensedFinancialStatementsTable_lbl" />
    <link:label xml:lang="en-US" xlink:label="srt_ScheduleOfCondensedFinancialStatementsTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Condensed Financial Statements [Table]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/srt/2020/elts/srt-2020-01-31.xsd#srt_CondensedFinancialStatementsCaptionsLineItems" xlink:type="locator" xlink:label="srt_CondensedFinancialStatementsCaptionsLineItems" />
    <link:label xml:lang="en-US" xlink:label="srt_CondensedFinancialStatementsCaptionsLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Condensed Financial Statements Captions [Line Items]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="srt_CondensedFinancialStatementsCaptionsLineItems" xlink:to="srt_CondensedFinancialStatementsCaptionsLineItems_lbl" />
    <link:label xml:lang="en-US" xlink:label="srt_CondensedFinancialStatementsCaptionsLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Condensed Financial Statements Captions [Line Items]</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_CashAndDueFromBanks_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Cash</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_InvestmentsInAffiliatesSubsidiariesAssociatesAndJointVentures" xlink:type="locator" xlink:label="us-gaap_InvestmentsInAffiliatesSubsidiariesAssociatesAndJointVentures" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InvestmentsInAffiliatesSubsidiariesAssociatesAndJointVentures_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Investment in bank subsidiary</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_InvestmentsInAffiliatesSubsidiariesAssociatesAndJointVentures" xlink:to="us-gaap_InvestmentsInAffiliatesSubsidiariesAssociatesAndJointVentures_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InvestmentsInAffiliatesSubsidiariesAssociatesAndJointVentures_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Investments In Affiliates Subsidiaries Associates And Joint Ventures</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_EquityMethodInvestmentAggregateCost" xlink:type="locator" xlink:label="us-gaap_EquityMethodInvestmentAggregateCost" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EquityMethodInvestmentAggregateCost_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Investment in non-bank subsidiary</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_EquityMethodInvestmentAggregateCost" xlink:to="us-gaap_EquityMethodInvestmentAggregateCost_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_EquityMethodInvestmentAggregateCost_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Equity Method Investment Aggregate Cost</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AccruedLiabilitiesCurrentAndNoncurrent" xlink:type="locator" xlink:label="us-gaap_AccruedLiabilitiesCurrentAndNoncurrent" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AccruedLiabilitiesCurrentAndNoncurrent_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Accrued liabilities</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AccruedLiabilitiesCurrentAndNoncurrent" xlink:to="us-gaap_AccruedLiabilitiesCurrentAndNoncurrent_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AccruedLiabilitiesCurrentAndNoncurrent_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Accrued Liabilities Current And Noncurrent</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_StockholdersEquity_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Shareholders&apos; equity</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ExpensesInterestExpenseAndOtherOperatingExpenses" xlink:type="locator" xlink:label="pbhc_ExpensesInterestExpenseAndOtherOperatingExpenses" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ExpensesInterestExpenseAndOtherOperatingExpenses_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Total aggregate amount of all expense.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ExpensesInterestExpenseAndOtherOperatingExpenses" xlink:to="pbhc_ExpensesInterestExpenseAndOtherOperatingExpenses_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_RevenuesAbstract" xlink:type="locator" xlink:label="us-gaap_RevenuesAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RevenuesAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Income</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_RevenuesAbstract" xlink:to="us-gaap_RevenuesAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RevenuesAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Revenues [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_OtherInterestAndDividendIncome" xlink:type="locator" xlink:label="us-gaap_OtherInterestAndDividendIncome" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherInterestAndDividendIncome_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Dividends from non-bank subsidiary</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_OtherInterestAndDividendIncome" xlink:to="us-gaap_OtherInterestAndDividendIncome_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherInterestAndDividendIncome_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Other Interest And Dividend Income</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DividendIncomeEquitySecuritiesOperating" xlink:type="locator" xlink:label="us-gaap_DividendIncomeEquitySecuritiesOperating" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DividendIncomeEquitySecuritiesOperating_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Dividends from marketable equity security</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DividendIncomeEquitySecuritiesOperating" xlink:to="us-gaap_DividendIncomeEquitySecuritiesOperating_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DividendIncomeEquitySecuritiesOperating_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Dividend Income Equity Securities Operating</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_MarketableSecuritiesGainLoss" xlink:type="locator" xlink:label="us-gaap_MarketableSecuritiesGainLoss" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_MarketableSecuritiesGainLoss_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Gains on marketable equity securities</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_MarketableSecuritiesGainLoss" xlink:to="us-gaap_MarketableSecuritiesGainLoss_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_MarketableSecuritiesGainLoss_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Marketable Securities Gain Loss</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_NoninterestIncomeOtherOperatingIncome" xlink:type="locator" xlink:label="us-gaap_NoninterestIncomeOtherOperatingIncome" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_NoninterestIncomeOtherOperatingIncome_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Operating, net</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_NoninterestIncomeOtherOperatingIncome" xlink:to="us-gaap_NoninterestIncomeOtherOperatingIncome_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_NoninterestIncomeOtherOperatingIncome_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Noninterest Income Other Operating Income</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestAndDividendIncomeOperating_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Total income</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestExpense_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Interest</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherNoninterestExpense_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Operating, net</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ExpensesInterestExpenseAndOtherOperatingExpenses_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Total expenses</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ExpensesInterestExpenseAndOtherOperatingExpenses_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Expenses Interest Expense And Other Operating Expenses</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_IncomeLossFromContinuingOperationsBeforeIncomeTaxesMinorityInterestAndIncomeLossFromEquityMethodInvestments" xlink:type="locator" xlink:label="us-gaap_IncomeLossFromContinuingOperationsBeforeIncomeTaxesMinorityInterestAndIncomeLossFromEquityMethodInvestments" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncomeLossFromContinuingOperationsBeforeIncomeTaxesMinorityInterestAndIncomeLossFromEquityMethodInvestments_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Loss before taxes and equity in undistributed net income of subsidiaries</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_IncomeLossFromContinuingOperationsBeforeIncomeTaxesMinorityInterestAndIncomeLossFromEquityMethodInvestments" xlink:to="us-gaap_IncomeLossFromContinuingOperationsBeforeIncomeTaxesMinorityInterestAndIncomeLossFromEquityMethodInvestments_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncomeLossFromContinuingOperationsBeforeIncomeTaxesMinorityInterestAndIncomeLossFromEquityMethodInvestments_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Income Loss From Continuing Operations Before Income Taxes Minority Interest And Income Loss From Equity Method Investments</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncomeTaxExpenseBenefit_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Tax benefit</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_IncomeLossBeforeEquityInUndistributedEarningsOfSubsidiaries_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Loss before equity in undistributed net income of subsidiaries</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_IncomeLossBeforeEquityInUndistributedEarningsOfSubsidiaries_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Income Loss Before Equity In Undistributed Earnings Of Subsidiaries</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_EquityInUndistributedEarningsOfSubsidiaries_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Equity in undistributed net income of subsidiaries</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_EquityInUndistributedEarningsOfSubsidiaries_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Equity In Undistributed Earnings Of Subsidiaries</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_NetCashProvidedByUsedInOperatingActivitiesAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Operating Activities</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_NetIncomeLoss_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Net Income</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_EquityInUndistributedEarningsOfSubsidiaries_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Equity in undistributed net income of subsidiaries</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AmortizationOfFinancingCostsAndDiscounts" xlink:type="locator" xlink:label="us-gaap_AmortizationOfFinancingCostsAndDiscounts" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AmortizationOfFinancingCostsAndDiscounts_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Amortization of deferred financing from subordinated loan</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AmortizationOfFinancingCostsAndDiscounts" xlink:to="us-gaap_AmortizationOfFinancingCostsAndDiscounts_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AmortizationOfFinancingCostsAndDiscounts_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Amortization Of Financing Costs And Discounts</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_NetCashProvidedByUsedInInvestingActivitiesAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Investing Activities</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_PaymentsToAcquireInterestInSubsidiariesAndAffiliates" xlink:type="locator" xlink:label="us-gaap_PaymentsToAcquireInterestInSubsidiariesAndAffiliates" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PaymentsToAcquireInterestInSubsidiariesAndAffiliates_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Capital contributed to wholly-owned bank subsidiary</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_PaymentsToAcquireInterestInSubsidiariesAndAffiliates" xlink:to="us-gaap_PaymentsToAcquireInterestInSubsidiariesAndAffiliates_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PaymentsToAcquireInterestInSubsidiariesAndAffiliates_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Payments To Acquire Interest In Subsidiaries And Affiliates</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_NetCashProvidedByUsedInFinancingActivitiesAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Financing activities</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ProceedsFromJuniorSubordinatedDebentureOwedToUnconsolidatedSubsidiaryTrust_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Proceeds from subordinated debt offering</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ScheduleOfRelatedPartyTransactionsByRelatedPartyTable" xlink:type="locator" xlink:label="us-gaap_ScheduleOfRelatedPartyTransactionsByRelatedPartyTable" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfRelatedPartyTransactionsByRelatedPartyTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Schedule Of Related Party Transactions By Related Party [Table]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ScheduleOfRelatedPartyTransactionsByRelatedPartyTable" xlink:to="us-gaap_ScheduleOfRelatedPartyTransactionsByRelatedPartyTable_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ScheduleOfRelatedPartyTransactionsByRelatedPartyTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Schedule Of Related Party Transactions By Related Party [Table]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_RelatedPartyTransactionsByRelatedPartyAxis" xlink:type="locator" xlink:label="us-gaap_RelatedPartyTransactionsByRelatedPartyAxis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RelatedPartyTransactionsByRelatedPartyAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Related Party</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_RelatedPartyTransactionsByRelatedPartyAxis" xlink:to="us-gaap_RelatedPartyTransactionsByRelatedPartyAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RelatedPartyTransactionsByRelatedPartyAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Related Party Transactions By Related Party [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_RelatedPartyDomain" xlink:type="locator" xlink:label="us-gaap_RelatedPartyDomain" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RelatedPartyDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Related Party</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_RelatedPartyDomain" xlink:to="us-gaap_RelatedPartyDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RelatedPartyDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Related Party [Domain]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_DirectorsExecutiveOfficersAndAffiliatesMember" xlink:type="locator" xlink:label="pbhc_DirectorsExecutiveOfficersAndAffiliatesMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DirectorsExecutiveOfficersAndAffiliatesMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Ranking officers of the entity, appointed to the position by the board of directors, people serving on the board of directors (who collectively have responsibility for governing the entity) and affiliates.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_DirectorsExecutiveOfficersAndAffiliatesMember" xlink:to="pbhc_DirectorsExecutiveOfficersAndAffiliatesMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DirectorsExecutiveOfficersAndAffiliatesMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Directors, Executive Officers and Affiliates [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DirectorsExecutiveOfficersAndAffiliatesMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Directors Executive Officers And Affiliates [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_RelatedPartyTransactionLineItems" xlink:type="locator" xlink:label="us-gaap_RelatedPartyTransactionLineItems" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RelatedPartyTransactionLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Related Party Transaction [Line Items]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_RelatedPartyTransactionLineItems" xlink:to="us-gaap_RelatedPartyTransactionLineItems_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RelatedPartyTransactionLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Related Party Transaction [Line Items]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LoansAndLeasesReceivableRelatedPartiesRollForward" xlink:type="locator" xlink:label="us-gaap_LoansAndLeasesReceivableRelatedPartiesRollForward" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LoansAndLeasesReceivableRelatedPartiesRollForward_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Loans to Related Parties [Roll Forward]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LoansAndLeasesReceivableRelatedPartiesRollForward" xlink:to="us-gaap_LoansAndLeasesReceivableRelatedPartiesRollForward_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LoansAndLeasesReceivableRelatedPartiesRollForward_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Loans And Leases Receivable Related Parties Roll Forward</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LoansAndLeasesReceivableRelatedParties" xlink:type="locator" xlink:label="us-gaap_LoansAndLeasesReceivableRelatedParties" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LoansAndLeasesReceivableRelatedParties_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/periodStartLabel">Balance at the beginning of the year</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LoansAndLeasesReceivableRelatedParties" xlink:to="us-gaap_LoansAndLeasesReceivableRelatedParties_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LoansAndLeasesReceivableRelatedParties_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Loans And Leases Receivable Related Parties</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LoansAndLeasesReceivableRelatedPartiesAdditions" xlink:type="locator" xlink:label="us-gaap_LoansAndLeasesReceivableRelatedPartiesAdditions" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LoansAndLeasesReceivableRelatedPartiesAdditions_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Originations and related party additions</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LoansAndLeasesReceivableRelatedPartiesAdditions" xlink:to="us-gaap_LoansAndLeasesReceivableRelatedPartiesAdditions_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LoansAndLeasesReceivableRelatedPartiesAdditions_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Loans And Leases Receivable Related Parties Additions</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LoansAndLeasesReceivableRelatedPartiesProceeds" xlink:type="locator" xlink:label="us-gaap_LoansAndLeasesReceivableRelatedPartiesProceeds" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LoansAndLeasesReceivableRelatedPartiesProceeds_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Principal payments and related party removals</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LoansAndLeasesReceivableRelatedPartiesProceeds" xlink:to="us-gaap_LoansAndLeasesReceivableRelatedPartiesProceeds_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LoansAndLeasesReceivableRelatedPartiesProceeds_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Loans And Leases Receivable Related Parties Proceeds</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_LoansAndLeasesReceivableRelatedParties_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/periodEndLabel">Balance at the end of the year</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_PurchasePriceOfProperty" xlink:type="locator" xlink:label="pbhc_PurchasePriceOfProperty" />
    <link:label xml:lang="en-US" xlink:label="pbhc_PurchasePriceOfProperty_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Purchase price of property</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_PurchasePriceOfProperty" xlink:to="pbhc_PurchasePriceOfProperty_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ProjectDevelopmentCost" xlink:type="locator" xlink:label="pbhc_ProjectDevelopmentCost" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ProjectDevelopmentCost_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Project development cost.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ProjectDevelopmentCost" xlink:to="pbhc_ProjectDevelopmentCost_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_LeaseCommencementDate" xlink:type="locator" xlink:label="pbhc_LeaseCommencementDate" />
    <link:label xml:lang="en-US" xlink:label="pbhc_LeaseCommencementDate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Lease commencement date.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_LeaseCommencementDate" xlink:to="pbhc_LeaseCommencementDate_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_RelatedPartyTransactionAxis" xlink:type="locator" xlink:label="us-gaap_RelatedPartyTransactionAxis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RelatedPartyTransactionAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Related Party Transaction</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_RelatedPartyTransactionAxis" xlink:to="us-gaap_RelatedPartyTransactionAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RelatedPartyTransactionAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Related Party Transaction [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_RelatedPartyTransactionDomain" xlink:type="locator" xlink:label="us-gaap_RelatedPartyTransactionDomain" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RelatedPartyTransactionDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Related Party Transaction</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_RelatedPartyTransactionDomain" xlink:to="us-gaap_RelatedPartyTransactionDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RelatedPartyTransactionDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Related Party Transaction [Domain]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LimitedLiabilityCompanyMember" xlink:type="locator" xlink:label="us-gaap_LimitedLiabilityCompanyMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LimitedLiabilityCompanyMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Limited Liability Company [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LimitedLiabilityCompanyMember" xlink:to="us-gaap_LimitedLiabilityCompanyMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LimitedLiabilityCompanyMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Limited Liability Company [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ConstructionMember" xlink:type="locator" xlink:label="us-gaap_ConstructionMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ConstructionMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Construction [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ConstructionMember" xlink:to="us-gaap_ConstructionMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ConstructionMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Construction [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_RelatedPartyDepositLiabilities" xlink:type="locator" xlink:label="us-gaap_RelatedPartyDepositLiabilities" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RelatedPartyDepositLiabilities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Deposits of related parties</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_RelatedPartyDepositLiabilities" xlink:to="us-gaap_RelatedPartyDepositLiabilities_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_RelatedPartyDepositLiabilities_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Related Party Deposit Liabilities</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_PurchasePriceOfProperty_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Purchase price of property</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_PurchasePriceOfProperty_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Purchase Price Of Property</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ProjectDevelopmentCost_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Total project cost</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ProjectDevelopmentCost_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Project Development Cost</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LesseeOperatingLeaseTermOfContract" xlink:type="locator" xlink:label="us-gaap_LesseeOperatingLeaseTermOfContract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LesseeOperatingLeaseTermOfContract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Lease term of contract</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LesseeOperatingLeaseTermOfContract" xlink:to="us-gaap_LesseeOperatingLeaseTermOfContract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LesseeOperatingLeaseTermOfContract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Lessee Operating Lease Term Of Contract</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_LeaseCommencementDate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Lease commencement date</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_LeaseCommencementDate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Lease Commencement Date</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_OperatingLeasesFutureMinimumPaymentsDueCurrent" xlink:type="locator" xlink:label="us-gaap_OperatingLeasesFutureMinimumPaymentsDueCurrent" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OperatingLeasesFutureMinimumPaymentsDueCurrent_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Lease rental payments for first twelve months</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_OperatingLeasesFutureMinimumPaymentsDueCurrent" xlink:to="us-gaap_OperatingLeasesFutureMinimumPaymentsDueCurrent_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OperatingLeasesFutureMinimumPaymentsDueCurrent_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Operating Leases Future Minimum Payments Due Current</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_OperatingLeasesFutureMinimumPaymentsDueThereafter" xlink:type="locator" xlink:label="us-gaap_OperatingLeasesFutureMinimumPaymentsDueThereafter" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OperatingLeasesFutureMinimumPaymentsDueThereafter_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Lease rental payments subsequently</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_OperatingLeasesFutureMinimumPaymentsDueThereafter" xlink:to="us-gaap_OperatingLeasesFutureMinimumPaymentsDueThereafter_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OperatingLeasesFutureMinimumPaymentsDueThereafter_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Operating Leases Future Minimum Payments Due Thereafter</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_GrossProceedsFromIssuanceOfCommonStock" xlink:type="locator" xlink:label="pbhc_GrossProceedsFromIssuanceOfCommonStock" />
    <link:label xml:lang="en-US" xlink:label="pbhc_GrossProceedsFromIssuanceOfCommonStock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Gross cash inflow from the additional capital contribution to the entity.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_GrossProceedsFromIssuanceOfCommonStock" xlink:to="pbhc_GrossProceedsFromIssuanceOfCommonStock_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ConversionOfStockTable" xlink:type="locator" xlink:label="us-gaap_ConversionOfStockTable" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ConversionOfStockTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Conversion Of Stock [Table]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ConversionOfStockTable" xlink:to="us-gaap_ConversionOfStockTable_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ConversionOfStockTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Conversion Of Stock [Table]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ConversionOfStockLineItems" xlink:type="locator" xlink:label="us-gaap_ConversionOfStockLineItems" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ConversionOfStockLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Conversion Of Stock [Line Items]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ConversionOfStockLineItems" xlink:to="us-gaap_ConversionOfStockLineItems_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ConversionOfStockLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Conversion Of Stock [Line Items]</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_EquityMethodInvestmentOwnershipPercentage_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Percentage owned by holding company</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_SharesIssuedPricePerShare_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Common stock sold per share (in dollar per share)</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_GrossProceedsFromIssuanceOfCommonStock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Gross offering proceeds of common stock</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_GrossProceedsFromIssuanceOfCommonStock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Gross Proceeds From Issuance Of Common Stock</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_CashAcquiredFromAcquisition" xlink:type="locator" xlink:label="us-gaap_CashAcquiredFromAcquisition" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CashAcquiredFromAcquisition_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Cash received from acquisition</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_CashAcquiredFromAcquisition" xlink:to="us-gaap_CashAcquiredFromAcquisition_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CashAcquiredFromAcquisition_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Cash Acquired From Acquisition</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_PaymentsOfStockIssuanceCosts" xlink:type="locator" xlink:label="us-gaap_PaymentsOfStockIssuanceCosts" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PaymentsOfStockIssuanceCosts_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Cost related to offering</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_PaymentsOfStockIssuanceCosts" xlink:to="us-gaap_PaymentsOfStockIssuanceCosts_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PaymentsOfStockIssuanceCosts_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Payments Of Stock Issuance Costs</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_ProceedsFromIssuanceOfCommonStock_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Net offering proceeds of common stock</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AccumulatedOtherComprehensiveIncomeLossTable" xlink:type="locator" xlink:label="us-gaap_AccumulatedOtherComprehensiveIncomeLossTable" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AccumulatedOtherComprehensiveIncomeLossTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Accumulated Other Comprehensive Income Loss [Table]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AccumulatedOtherComprehensiveIncomeLossTable" xlink:to="us-gaap_AccumulatedOtherComprehensiveIncomeLossTable_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AccumulatedOtherComprehensiveIncomeLossTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Accumulated Other Comprehensive Income Loss [Table]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AccumulatedDefinedBenefitPlansAdjustmentIncludingPortionAttributableToNoncontrollingInterestMember" xlink:type="locator" xlink:label="us-gaap_AccumulatedDefinedBenefitPlansAdjustmentIncludingPortionAttributableToNoncontrollingInterestMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AccumulatedDefinedBenefitPlansAdjustmentIncludingPortionAttributableToNoncontrollingInterestMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Retirement Plans [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AccumulatedDefinedBenefitPlansAdjustmentIncludingPortionAttributableToNoncontrollingInterestMember" xlink:to="us-gaap_AccumulatedDefinedBenefitPlansAdjustmentIncludingPortionAttributableToNoncontrollingInterestMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AccumulatedDefinedBenefitPlansAdjustmentIncludingPortionAttributableToNoncontrollingInterestMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Accumulated Defined Benefit Plans Adjustment Including Portion Attributable To Noncontrolling Interest [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AccumulatedNetInvestmentGainLossIncludingPortionAttributableToNoncontrollingInterestMember" xlink:type="locator" xlink:label="us-gaap_AccumulatedNetInvestmentGainLossIncludingPortionAttributableToNoncontrollingInterestMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AccumulatedNetInvestmentGainLossIncludingPortionAttributableToNoncontrollingInterestMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Unrealized Gains and Losses on Available-for-Sale Securities [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AccumulatedNetInvestmentGainLossIncludingPortionAttributableToNoncontrollingInterestMember" xlink:to="us-gaap_AccumulatedNetInvestmentGainLossIncludingPortionAttributableToNoncontrollingInterestMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AccumulatedNetInvestmentGainLossIncludingPortionAttributableToNoncontrollingInterestMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Accumulated Net Investment Gain Loss Including Portion Attributable To Noncontrolling Interest [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AccumulatedGainLossCashFlowHedgeIncludingNoncontrollingInterestMember" xlink:type="locator" xlink:label="us-gaap_AccumulatedGainLossCashFlowHedgeIncludingNoncontrollingInterestMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AccumulatedGainLossCashFlowHedgeIncludingNoncontrollingInterestMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Unrealized Losses on Derivatives and Hedging Activities [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AccumulatedGainLossCashFlowHedgeIncludingNoncontrollingInterestMember" xlink:to="us-gaap_AccumulatedGainLossCashFlowHedgeIncludingNoncontrollingInterestMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AccumulatedGainLossCashFlowHedgeIncludingNoncontrollingInterestMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Accumulated Gain Loss Cash Flow Hedge Including Noncontrolling Interest [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_AccumulatedNetUnrealizedInvestmentIncomeLossHeldToMaturitySecuritiesMember" xlink:type="locator" xlink:label="pbhc_AccumulatedNetUnrealizedInvestmentIncomeLossHeldToMaturitySecuritiesMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_AccumulatedNetUnrealizedInvestmentIncomeLossHeldToMaturitySecuritiesMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Changes to accumulated comprehensive income resulting from the net income or the net loss in value of securities transferred to held to maturity.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_AccumulatedNetUnrealizedInvestmentIncomeLossHeldToMaturitySecuritiesMember" xlink:to="pbhc_AccumulatedNetUnrealizedInvestmentIncomeLossHeldToMaturitySecuritiesMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_AccumulatedNetUnrealizedInvestmentIncomeLossHeldToMaturitySecuritiesMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Unrealized Loss on Securities Transferred to Held-to-Maturity [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AccumulatedNetUnrealizedInvestmentIncomeLossHeldToMaturitySecuritiesMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Accumulated Net Unrealized Investment Income Loss Held To Maturity Securities [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AociIncludingPortionAttributableToNoncontrollingInterestMember" xlink:type="locator" xlink:label="us-gaap_AociIncludingPortionAttributableToNoncontrollingInterestMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AociIncludingPortionAttributableToNoncontrollingInterestMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">AOCI Attributable to Parent [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AociIncludingPortionAttributableToNoncontrollingInterestMember" xlink:to="us-gaap_AociIncludingPortionAttributableToNoncontrollingInterestMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AociIncludingPortionAttributableToNoncontrollingInterestMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Aoci Including Portion Attributable To Noncontrolling Interest [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AccumulatedOtherComprehensiveIncomeLossLineItems" xlink:type="locator" xlink:label="us-gaap_AccumulatedOtherComprehensiveIncomeLossLineItems" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AccumulatedOtherComprehensiveIncomeLossLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Accumulated Other Comprehensive Income Loss [Line Items]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AccumulatedOtherComprehensiveIncomeLossLineItems" xlink:to="us-gaap_AccumulatedOtherComprehensiveIncomeLossLineItems_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AccumulatedOtherComprehensiveIncomeLossLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Accumulated Other Comprehensive Income Loss [Line Items]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_AOCIIncludingPortionAttributableToNoncontrollingInterestAbstract" xlink:type="locator" xlink:label="us-gaap_AOCIIncludingPortionAttributableToNoncontrollingInterestAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AOCIIncludingPortionAttributableToNoncontrollingInterestAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Changes in the Components of Accumulated Other Comprehensive Income (Loss), Net of Tax [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_AOCIIncludingPortionAttributableToNoncontrollingInterestAbstract" xlink:to="us-gaap_AOCIIncludingPortionAttributableToNoncontrollingInterestAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_AOCIIncludingPortionAttributableToNoncontrollingInterestAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">A O C I Including Portion Attributable To Noncontrolling Interest [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_OtherComprehensiveIncomeLossBeforeReclassificationsNetOfTax" xlink:type="locator" xlink:label="us-gaap_OtherComprehensiveIncomeLossBeforeReclassificationsNetOfTax" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherComprehensiveIncomeLossBeforeReclassificationsNetOfTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Other comprehensive income before reclassifications</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_OtherComprehensiveIncomeLossBeforeReclassificationsNetOfTax" xlink:to="us-gaap_OtherComprehensiveIncomeLossBeforeReclassificationsNetOfTax_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OtherComprehensiveIncomeLossBeforeReclassificationsNetOfTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Other Comprehensive Income Loss Before Reclassifications Net Of Tax</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ReclassificationFromAccumulatedOtherComprehensiveIncomeCurrentPeriodNetOfTax" xlink:type="locator" xlink:label="us-gaap_ReclassificationFromAccumulatedOtherComprehensiveIncomeCurrentPeriodNetOfTax" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ReclassificationFromAccumulatedOtherComprehensiveIncomeCurrentPeriodNetOfTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Amounts reclassified from AOCI</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ReclassificationFromAccumulatedOtherComprehensiveIncomeCurrentPeriodNetOfTax" xlink:to="us-gaap_ReclassificationFromAccumulatedOtherComprehensiveIncomeCurrentPeriodNetOfTax_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ReclassificationFromAccumulatedOtherComprehensiveIncomeCurrentPeriodNetOfTax_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Reclassification From Accumulated Other Comprehensive Income Current Period Net Of Tax</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ReclassificationOutOfAccumulatedOtherComprehensiveIncomeTable" xlink:type="locator" xlink:label="us-gaap_ReclassificationOutOfAccumulatedOtherComprehensiveIncomeTable" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ReclassificationOutOfAccumulatedOtherComprehensiveIncomeTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Reclassification Out Of Accumulated Other Comprehensive Income [Table]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ReclassificationOutOfAccumulatedOtherComprehensiveIncomeTable" xlink:to="us-gaap_ReclassificationOutOfAccumulatedOtherComprehensiveIncomeTable_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ReclassificationOutOfAccumulatedOtherComprehensiveIncomeTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Reclassification Out Of Accumulated Other Comprehensive Income [Table]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ReclassificationOutOfAccumulatedOtherComprehensiveIncomeAxis" xlink:type="locator" xlink:label="us-gaap_ReclassificationOutOfAccumulatedOtherComprehensiveIncomeAxis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ReclassificationOutOfAccumulatedOtherComprehensiveIncomeAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Reclassification out of Accumulated Other Comprehensive Income</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ReclassificationOutOfAccumulatedOtherComprehensiveIncomeAxis" xlink:to="us-gaap_ReclassificationOutOfAccumulatedOtherComprehensiveIncomeAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ReclassificationOutOfAccumulatedOtherComprehensiveIncomeAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Reclassification Out Of Accumulated Other Comprehensive Income [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ReclassificationOutOfAccumulatedOtherComprehensiveIncomeDomain" xlink:type="locator" xlink:label="us-gaap_ReclassificationOutOfAccumulatedOtherComprehensiveIncomeDomain" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ReclassificationOutOfAccumulatedOtherComprehensiveIncomeDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Reclassification out of Accumulated Other Comprehensive Income</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ReclassificationOutOfAccumulatedOtherComprehensiveIncomeDomain" xlink:to="us-gaap_ReclassificationOutOfAccumulatedOtherComprehensiveIncomeDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ReclassificationOutOfAccumulatedOtherComprehensiveIncomeDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Reclassification Out Of Accumulated Other Comprehensive Income [Domain]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ReclassificationOutOfAccumulatedOtherComprehensiveIncomeMember" xlink:type="locator" xlink:label="us-gaap_ReclassificationOutOfAccumulatedOtherComprehensiveIncomeMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ReclassificationOutOfAccumulatedOtherComprehensiveIncomeMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Reclassification out of Accumulated Other Comprehensive Income [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ReclassificationOutOfAccumulatedOtherComprehensiveIncomeMember" xlink:to="us-gaap_ReclassificationOutOfAccumulatedOtherComprehensiveIncomeMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ReclassificationOutOfAccumulatedOtherComprehensiveIncomeMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Reclassification Out Of Accumulated Other Comprehensive Income [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_AccumulatedDefinedBenefitPlansAdjustmentIncludingPortionAttributableToNoncontrollingInterestMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Retirement Plans [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_AccumulatedNetInvestmentGainLossIncludingPortionAttributableToNoncontrollingInterestMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Available-for-Sale Securities [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_ReclassificationAdjustmentOutOfAccumulatedOtherComprehensiveIncomeLineItems" xlink:type="locator" xlink:label="us-gaap_ReclassificationAdjustmentOutOfAccumulatedOtherComprehensiveIncomeLineItems" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ReclassificationAdjustmentOutOfAccumulatedOtherComprehensiveIncomeLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Reclassification Adjustment Out Of Accumulated Other Comprehensive Income [Line Items]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_ReclassificationAdjustmentOutOfAccumulatedOtherComprehensiveIncomeLineItems" xlink:to="us-gaap_ReclassificationAdjustmentOutOfAccumulatedOtherComprehensiveIncomeLineItems_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_ReclassificationAdjustmentOutOfAccumulatedOtherComprehensiveIncomeLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Reclassification Adjustment Out Of Accumulated Other Comprehensive Income [Line Items]</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_SalariesAndWages_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedLabel">Salaries and employee benefits</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DebtAndEquitySecuritiesGainLoss" xlink:type="locator" xlink:label="us-gaap_DebtAndEquitySecuritiesGainLoss" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtAndEquitySecuritiesGainLoss_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Net gains on sales and redemptions of investment securities</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DebtAndEquitySecuritiesGainLoss" xlink:to="us-gaap_DebtAndEquitySecuritiesGainLoss_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DebtAndEquitySecuritiesGainLoss_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Debt And Equity Securities Gain Loss</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncomeTaxExpenseBenefit_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel">Provision for income taxes</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DisaggregationOfRevenueTable" xlink:type="locator" xlink:label="us-gaap_DisaggregationOfRevenueTable" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DisaggregationOfRevenueTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Disaggregation Of Revenue [Table]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DisaggregationOfRevenueTable" xlink:to="us-gaap_DisaggregationOfRevenueTable_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DisaggregationOfRevenueTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Disaggregation Of Revenue [Table]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_InsufficientFundsFeesDepositAccountMember" xlink:type="locator" xlink:label="pbhc_InsufficientFundsFeesDepositAccountMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_InsufficientFundsFeesDepositAccountMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Insufficient funds fees deposit account.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_InsufficientFundsFeesDepositAccountMember" xlink:to="pbhc_InsufficientFundsFeesDepositAccountMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_InsufficientFundsFeesDepositAccountMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Insufficient Funds Fees [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_InsufficientFundsFeesDepositAccountMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Insufficient Funds Fees Deposit Account [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_DepositRelatedFeesDepositAccountMember" xlink:type="locator" xlink:label="pbhc_DepositRelatedFeesDepositAccountMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DepositRelatedFeesDepositAccountMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Deposit related fees deposit account.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_DepositRelatedFeesDepositAccountMember" xlink:to="pbhc_DepositRelatedFeesDepositAccountMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DepositRelatedFeesDepositAccountMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Deposit Related Fees [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DepositRelatedFeesDepositAccountMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deposit Related Fees Deposit Account [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ATMFeesDepositAccountMember" xlink:type="locator" xlink:label="pbhc_ATMFeesDepositAccountMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ATMFeesDepositAccountMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">ATM fees deposit account.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ATMFeesDepositAccountMember" xlink:to="pbhc_ATMFeesDepositAccountMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ATMFeesDepositAccountMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">ATM Fees [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ATMFeesDepositAccountMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">A T M Fees Deposit Account [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_DepositAccountMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Service Fees [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_InsuranceCommissionsMember" xlink:type="locator" xlink:label="pbhc_InsuranceCommissionsMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_InsuranceCommissionsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Insurance commissions.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_InsuranceCommissionsMember" xlink:to="pbhc_InsuranceCommissionsMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_InsuranceCommissionsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Insurance Commissions [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_InsuranceCommissionsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Insurance Commissions [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_InvestmentServicesRevenueMember" xlink:type="locator" xlink:label="pbhc_InvestmentServicesRevenueMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_InvestmentServicesRevenueMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Investment services revenue.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_InvestmentServicesRevenueMember" xlink:to="pbhc_InvestmentServicesRevenueMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_InvestmentServicesRevenueMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Investment Services Revenue [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_InvestmentServicesRevenueMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Investment Services Revenue [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ATMFeesSurchargeMember" xlink:type="locator" xlink:label="pbhc_ATMFeesSurchargeMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ATMFeesSurchargeMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">ATM fees surcharge.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ATMFeesSurchargeMember" xlink:to="pbhc_ATMFeesSurchargeMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ATMFeesSurchargeMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">ATM Fees Surcharge [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ATMFeesSurchargeMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">A T M Fees Surcharge [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_BankingHouseRentsCollectedMember" xlink:type="locator" xlink:label="pbhc_BankingHouseRentsCollectedMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_BankingHouseRentsCollectedMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Banking house rents collected.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_BankingHouseRentsCollectedMember" xlink:to="pbhc_BankingHouseRentsCollectedMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_BankingHouseRentsCollectedMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Banking House Rents Collected [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_BankingHouseRentsCollectedMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Banking House Rents Collected [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_FeeIncomeMember" xlink:type="locator" xlink:label="pbhc_FeeIncomeMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_FeeIncomeMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Fee income.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_FeeIncomeMember" xlink:to="pbhc_FeeIncomeMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_FeeIncomeMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Fee Income [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_FeeIncomeMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Fee Income [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_CreditCardMerchantDiscountMember" xlink:type="locator" xlink:label="us-gaap_CreditCardMerchantDiscountMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CreditCardMerchantDiscountMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Merchant Card Fees [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_CreditCardMerchantDiscountMember" xlink:to="us-gaap_CreditCardMerchantDiscountMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CreditCardMerchantDiscountMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Credit Card Merchant Discount [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_CreditAndDebitCardMember" xlink:type="locator" xlink:label="us-gaap_CreditAndDebitCardMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CreditAndDebitCardMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Card Income [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_CreditAndDebitCardMember" xlink:to="us-gaap_CreditAndDebitCardMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CreditAndDebitCardMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Credit And Debit Card [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_NetGainsLossesOnSaleOfLoansAndForeclosedRealEstateMember" xlink:type="locator" xlink:label="pbhc_NetGainsLossesOnSaleOfLoansAndForeclosedRealEstateMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NetGainsLossesOnSaleOfLoansAndForeclosedRealEstateMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Net gains losses on sale of loans and foreclosed real estate.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_NetGainsLossesOnSaleOfLoansAndForeclosedRealEstateMember" xlink:to="pbhc_NetGainsLossesOnSaleOfLoansAndForeclosedRealEstateMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NetGainsLossesOnSaleOfLoansAndForeclosedRealEstateMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Net Gains (Losses) on Sale of Loans and Foreclosed Real Estate [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NetGainsLossesOnSaleOfLoansAndForeclosedRealEstateMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Net Gains Losses On Sale Of Loans And Foreclosed Real Estate [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_MortgageBankingMember" xlink:type="locator" xlink:label="us-gaap_MortgageBankingMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_MortgageBankingMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Total Mortgage Fee Income and Realized Gain on Sale of Loans and Foreclosed Real Estate [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_MortgageBankingMember" xlink:to="us-gaap_MortgageBankingMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_MortgageBankingMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Mortgage Banking [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_BankingMember" xlink:type="locator" xlink:label="us-gaap_BankingMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_BankingMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Service Fees, Fee Income, Card Income and Mortgage Fee Income and Realized Gain on Sale of Loans and Foreclosed Real Estate [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_BankingMember" xlink:to="us-gaap_BankingMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_BankingMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Banking [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DisaggregationOfRevenueLineItems" xlink:type="locator" xlink:label="us-gaap_DisaggregationOfRevenueLineItems" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DisaggregationOfRevenueLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Disaggregation Of Revenue [Line Items]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DisaggregationOfRevenueLineItems" xlink:to="us-gaap_DisaggregationOfRevenueLineItems_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DisaggregationOfRevenueLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Disaggregation Of Revenue [Line Items]</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_GainLossOnSaleOfInvestments_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Net gains on sales and redemptions of investment securities</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_NoninterestIncomeOther" xlink:type="locator" xlink:label="us-gaap_NoninterestIncomeOther" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_NoninterestIncomeOther_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Other miscellaneous income</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_NoninterestIncomeOther" xlink:to="us-gaap_NoninterestIncomeOther_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_NoninterestIncomeOther_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Noninterest Income Other</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_LesseeOperatingLeasesRemainingLeaseTerm" xlink:type="locator" xlink:label="pbhc_LesseeOperatingLeasesRemainingLeaseTerm" />
    <link:label xml:lang="en-US" xlink:label="pbhc_LesseeOperatingLeasesRemainingLeaseTerm_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Lessee, operating leases remaining lease term.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_LesseeOperatingLeasesRemainingLeaseTerm" xlink:to="pbhc_LesseeOperatingLeasesRemainingLeaseTerm_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LesseeLeaseDescriptionTable" xlink:type="locator" xlink:label="us-gaap_LesseeLeaseDescriptionTable" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LesseeLeaseDescriptionTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Lessee Lease Description [Table]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LesseeLeaseDescriptionTable" xlink:to="us-gaap_LesseeLeaseDescriptionTable_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LesseeLeaseDescriptionTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Lessee Lease Description [Table]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LesseeLeaseDescriptionLineItems" xlink:type="locator" xlink:label="us-gaap_LesseeLeaseDescriptionLineItems" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LesseeLeaseDescriptionLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Lessee Lease Description [Line Items]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LesseeLeaseDescriptionLineItems" xlink:to="us-gaap_LesseeLeaseDescriptionLineItems_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LesseeLeaseDescriptionLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Lessee Lease Description [Line Items]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_LesseeOperatingLeasesRemainingLeaseTerm_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Operating leases remaining lease term</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_LesseeOperatingLeasesRemainingLeaseTerm_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Lessee Operating Leases Remaining Lease Term</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LesseeOperatingLeaseOptionToExtend" xlink:type="locator" xlink:label="us-gaap_LesseeOperatingLeaseOptionToExtend" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LesseeOperatingLeaseOptionToExtend_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Operating lease, option to extend lease term, description</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LesseeOperatingLeaseOptionToExtend" xlink:to="us-gaap_LesseeOperatingLeaseOptionToExtend_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LesseeOperatingLeaseOptionToExtend_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Lessee Operating Lease Option To Extend</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LesseeOperatingLeaseExistenceOfOptionToExtend" xlink:type="locator" xlink:label="us-gaap_LesseeOperatingLeaseExistenceOfOptionToExtend" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LesseeOperatingLeaseExistenceOfOptionToExtend_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Operating lease, option to extend lease term</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LesseeOperatingLeaseExistenceOfOptionToExtend" xlink:to="us-gaap_LesseeOperatingLeaseExistenceOfOptionToExtend_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LesseeOperatingLeaseExistenceOfOptionToExtend_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Lessee Operating Lease Existence Of Option To Extend</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_FinanceLeaseCost" xlink:type="locator" xlink:label="pbhc_FinanceLeaseCost" />
    <link:label xml:lang="en-US" xlink:label="pbhc_FinanceLeaseCost_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Finance lease cost.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_FinanceLeaseCost" xlink:to="pbhc_FinanceLeaseCost_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_OperatingLeaseCost" xlink:type="locator" xlink:label="us-gaap_OperatingLeaseCost" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OperatingLeaseCost_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Operating lease cost</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_OperatingLeaseCost" xlink:to="us-gaap_OperatingLeaseCost_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OperatingLeaseCost_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Operating Lease Cost</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_FinanceLeaseCost_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Finance lease cost</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_FinanceLeaseCost_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Finance Lease Cost</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_CashPaidForAmountsIncludedInMeasurementOfLeaseLiabilitiesAbstract" xlink:type="locator" xlink:label="pbhc_CashPaidForAmountsIncludedInMeasurementOfLeaseLiabilitiesAbstract" />
    <link:label xml:lang="en-US" xlink:label="pbhc_CashPaidForAmountsIncludedInMeasurementOfLeaseLiabilitiesAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Cash paid for amounts included in the measurement of lease liabilities.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_CashPaidForAmountsIncludedInMeasurementOfLeaseLiabilitiesAbstract" xlink:to="pbhc_CashPaidForAmountsIncludedInMeasurementOfLeaseLiabilitiesAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_CashPaidForAmountsIncludedInMeasurementOfLeaseLiabilitiesAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Cash paid for amount included in the measurement of lease liabilities:</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_CashPaidForAmountsIncludedInMeasurementOfLeaseLiabilitiesAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Cash Paid For Amounts Included In Measurement Of Lease Liabilities [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_OperatingLeasePayments" xlink:type="locator" xlink:label="us-gaap_OperatingLeasePayments" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OperatingLeasePayments_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Operating cash flows from operating leases</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_OperatingLeasePayments" xlink:to="us-gaap_OperatingLeasePayments_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OperatingLeasePayments_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Operating Lease Payments</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FinanceLeaseInterestPaymentOnLiability" xlink:type="locator" xlink:label="us-gaap_FinanceLeaseInterestPaymentOnLiability" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinanceLeaseInterestPaymentOnLiability_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Operating cash flows from finance leases</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FinanceLeaseInterestPaymentOnLiability" xlink:to="us-gaap_FinanceLeaseInterestPaymentOnLiability_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinanceLeaseInterestPaymentOnLiability_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Finance Lease Interest Payment On Liability</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FinanceLeasePrincipalPayments" xlink:type="locator" xlink:label="us-gaap_FinanceLeasePrincipalPayments" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinanceLeasePrincipalPayments_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Financing cash flows from finance leases</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FinanceLeasePrincipalPayments" xlink:to="us-gaap_FinanceLeasePrincipalPayments_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinanceLeasePrincipalPayments_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Finance Lease Principal Payments</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_FinanceLeasesAbstract" xlink:type="locator" xlink:label="pbhc_FinanceLeasesAbstract" />
    <link:label xml:lang="en-US" xlink:label="pbhc_FinanceLeasesAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Finance leases.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_FinanceLeasesAbstract" xlink:to="pbhc_FinanceLeasesAbstract_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_WeightedAverageRemainingLeaseTermAbstract" xlink:type="locator" xlink:label="pbhc_WeightedAverageRemainingLeaseTermAbstract" />
    <link:label xml:lang="en-US" xlink:label="pbhc_WeightedAverageRemainingLeaseTermAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Weighted average remaining lease term.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_WeightedAverageRemainingLeaseTermAbstract" xlink:to="pbhc_WeightedAverageRemainingLeaseTermAbstract_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_WeightedAverageDiscountRateAbstract" xlink:type="locator" xlink:label="pbhc_WeightedAverageDiscountRateAbstract" />
    <link:label xml:lang="en-US" xlink:label="pbhc_WeightedAverageDiscountRateAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Weighted average discount rate.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_WeightedAverageDiscountRateAbstract" xlink:to="pbhc_WeightedAverageDiscountRateAbstract_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LeasesOperatingAbstract" xlink:type="locator" xlink:label="us-gaap_LeasesOperatingAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LeasesOperatingAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Operating Leases:</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LeasesOperatingAbstract" xlink:to="us-gaap_LeasesOperatingAbstract_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LeasesOperatingAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Leases Operating [Abstract]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_FinanceLeasesAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Finance Leases:</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_FinanceLeasesAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Finance Leases [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FinanceLeaseLiability" xlink:type="locator" xlink:label="us-gaap_FinanceLeaseLiability" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinanceLeaseLiability_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Financial Liability</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FinanceLeaseLiability" xlink:to="us-gaap_FinanceLeaseLiability_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinanceLeaseLiability_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Finance Lease Liability</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_WeightedAverageRemainingLeaseTermAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Weighted Average Remaining Lease Term:</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_WeightedAverageRemainingLeaseTermAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Weighted Average Remaining Lease Term [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_OperatingLeaseWeightedAverageRemainingLeaseTerm1" xlink:type="locator" xlink:label="us-gaap_OperatingLeaseWeightedAverageRemainingLeaseTerm1" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OperatingLeaseWeightedAverageRemainingLeaseTerm1_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Operating Leases</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_OperatingLeaseWeightedAverageRemainingLeaseTerm1" xlink:to="us-gaap_OperatingLeaseWeightedAverageRemainingLeaseTerm1_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OperatingLeaseWeightedAverageRemainingLeaseTerm1_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Operating Lease Weighted Average Remaining Lease Term1</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_FinanceLeaseWeightedAverageRemainingLeaseTerm1" xlink:type="locator" xlink:label="us-gaap_FinanceLeaseWeightedAverageRemainingLeaseTerm1" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinanceLeaseWeightedAverageRemainingLeaseTerm1_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Finance Leases</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_FinanceLeaseWeightedAverageRemainingLeaseTerm1" xlink:to="us-gaap_FinanceLeaseWeightedAverageRemainingLeaseTerm1_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_FinanceLeaseWeightedAverageRemainingLeaseTerm1_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Finance Lease Weighted Average Remaining Lease Term1</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_WeightedAverageDiscountRateAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Weighted Average Discount Rate:</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_WeightedAverageDiscountRateAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Weighted Average Discount Rate [Abstract]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_OperatingLeaseWeightedAverageDiscountRatePercent" xlink:type="locator" xlink:label="us-gaap_OperatingLeaseWeightedAverageDiscountRatePercent" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OperatingLeaseWeightedAverageDiscountRatePercent_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Operating Leases</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_OperatingLeaseWeightedAverageDiscountRatePercent" xlink:to="us-gaap_OperatingLeaseWeightedAverageDiscountRatePercent_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OperatingLeaseWeightedAverageDiscountRatePercent_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Operating Lease Weighted Average Discount Rate Percent</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LesseeFinanceLeaseDiscountRate" xlink:type="locator" xlink:label="us-gaap_LesseeFinanceLeaseDiscountRate" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LesseeFinanceLeaseDiscountRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Finance Leases</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LesseeFinanceLeaseDiscountRate" xlink:to="us-gaap_LesseeFinanceLeaseDiscountRate_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LesseeFinanceLeaseDiscountRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Lessee Finance Lease Discount Rate</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_OperatingLeaseLiabilitiesPaymentsDueRollingMaturityAbstract" xlink:type="locator" xlink:label="us-gaap_OperatingLeaseLiabilitiesPaymentsDueRollingMaturityAbstract" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OperatingLeaseLiabilitiesPaymentsDueRollingMaturityAbstract_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Operating Lease Liabilities Payments Due Rolling Maturity [Abstract]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_OperatingLeaseLiabilitiesPaymentsDueRollingMaturityAbstract" xlink:to="us-gaap_OperatingLeaseLiabilitiesPaymentsDueRollingMaturityAbstract_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueNextRollingTwelveMonths" xlink:type="locator" xlink:label="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueNextRollingTwelveMonths" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueNextRollingTwelveMonths_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">2021</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueNextRollingTwelveMonths" xlink:to="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueNextRollingTwelveMonths_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueNextRollingTwelveMonths_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Lessee Operating Lease Liability Payments Due Next Rolling Twelve Months</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueInRollingYearTwo" xlink:type="locator" xlink:label="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueInRollingYearTwo" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueInRollingYearTwo_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">2022</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueInRollingYearTwo" xlink:to="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueInRollingYearTwo_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueInRollingYearTwo_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Lessee Operating Lease Liability Payments Due In Rolling Year Two</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueInRollingYearThree" xlink:type="locator" xlink:label="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueInRollingYearThree" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueInRollingYearThree_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">2023</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueInRollingYearThree" xlink:to="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueInRollingYearThree_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueInRollingYearThree_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Lessee Operating Lease Liability Payments Due In Rolling Year Three</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueInRollingYearFour" xlink:type="locator" xlink:label="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueInRollingYearFour" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueInRollingYearFour_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">2024</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueInRollingYearFour" xlink:to="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueInRollingYearFour_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueInRollingYearFour_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Lessee Operating Lease Liability Payments Due In Rolling Year Four</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueInRollingYearFive" xlink:type="locator" xlink:label="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueInRollingYearFive" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueInRollingYearFive_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">2025</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueInRollingYearFive" xlink:to="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueInRollingYearFive_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueInRollingYearFive_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Lessee Operating Lease Liability Payments Due In Rolling Year Five</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueAfterRollingYearFive" xlink:type="locator" xlink:label="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueAfterRollingYearFive" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueAfterRollingYearFive_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Thereafter</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueAfterRollingYearFive" xlink:to="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueAfterRollingYearFive_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueAfterRollingYearFive_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Lessee Operating Lease Liability Payments Due After Rolling Year Five</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LesseeOperatingLeaseLiabilityPaymentsDue" xlink:type="locator" xlink:label="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDue" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/totalLabel">Total minimum lease payments</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDue" xlink:to="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDue_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDue_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Lessee Operating Lease Liability Payments Due</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_OperatingLeaseLeaseIncome" xlink:type="locator" xlink:label="us-gaap_OperatingLeaseLeaseIncome" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OperatingLeaseLeaseIncome_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Lease rental income</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_OperatingLeaseLeaseIncome" xlink:to="us-gaap_OperatingLeaseLeaseIncome_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_OperatingLeaseLeaseIncome_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Operating Lease Lease Income</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_PercentageOfDecreaseInFederalFundsRateTarget" xlink:type="locator" xlink:label="pbhc_PercentageOfDecreaseInFederalFundsRateTarget" />
    <link:label xml:lang="en-US" xlink:label="pbhc_PercentageOfDecreaseInFederalFundsRateTarget_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Percentage of decrease in federal funds rate target.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_PercentageOfDecreaseInFederalFundsRateTarget" xlink:to="pbhc_PercentageOfDecreaseInFederalFundsRateTarget_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_BasisPointReductionInTargetFederalFundsRate" xlink:type="locator" xlink:label="pbhc_BasisPointReductionInTargetFederalFundsRate" />
    <link:label xml:lang="en-US" xlink:label="pbhc_BasisPointReductionInTargetFederalFundsRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Basis point reduction in target federal funds rate.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_BasisPointReductionInTargetFederalFundsRate" xlink:to="pbhc_BasisPointReductionInTargetFederalFundsRate_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_UnusualOrInfrequentItemTable" xlink:type="locator" xlink:label="us-gaap_UnusualOrInfrequentItemTable" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_UnusualOrInfrequentItemTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Unusual Or Infrequent Item [Table]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_UnusualOrInfrequentItemTable" xlink:to="us-gaap_UnusualOrInfrequentItemTable_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_UnusualOrInfrequentItemTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Unusual Or Infrequent Item [Table]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_TotalRelatedCreditsMember" xlink:type="locator" xlink:label="pbhc_TotalRelatedCreditsMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_TotalRelatedCreditsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Total related credits</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_TotalRelatedCreditsMember" xlink:to="pbhc_TotalRelatedCreditsMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_TotalRelatedCreditsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Total Related Credits [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_TotalRelatedCreditsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Total Related Credits [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_UnusualOrInfrequentItemMember" xlink:type="locator" xlink:label="us-gaap_UnusualOrInfrequentItemMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_UnusualOrInfrequentItemMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Unusual or Infrequent Item, or Both</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_UnusualOrInfrequentItemMember" xlink:to="us-gaap_UnusualOrInfrequentItemMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_UnusualOrInfrequentItemMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Unusual Or Infrequent Item [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_PPPLoanMember" xlink:type="locator" xlink:label="pbhc_PPPLoanMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_PPPLoanMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">PPP loan</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_PPPLoanMember" xlink:to="pbhc_PPPLoanMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_PPPLoanMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">PPP Loan [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_PPPLoanMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">P P P Loan [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LoanRestructuringModificationAxis" xlink:type="locator" xlink:label="us-gaap_LoanRestructuringModificationAxis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LoanRestructuringModificationAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Loan Restructuring Modification</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LoanRestructuringModificationAxis" xlink:to="us-gaap_LoanRestructuringModificationAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LoanRestructuringModificationAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Loan Restructuring Modification [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LoanRestructuringModificationDomain" xlink:type="locator" xlink:label="us-gaap_LoanRestructuringModificationDomain" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LoanRestructuringModificationDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Loan Restructuring Modification</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LoanRestructuringModificationDomain" xlink:to="us-gaap_LoanRestructuringModificationDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LoanRestructuringModificationDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Loan Restructuring Modification [Domain]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_PrincipalForgivenessMember" xlink:type="locator" xlink:label="us-gaap_PrincipalForgivenessMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PrincipalForgivenessMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Principal Forgiveness [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_PrincipalForgivenessMember" xlink:to="us-gaap_PrincipalForgivenessMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_PrincipalForgivenessMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Principal Forgiveness [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_IncomeStatementLocationAxis" xlink:type="locator" xlink:label="us-gaap_IncomeStatementLocationAxis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncomeStatementLocationAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Income Statement Location</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_IncomeStatementLocationAxis" xlink:to="us-gaap_IncomeStatementLocationAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncomeStatementLocationAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Income Statement Location [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_IncomeStatementLocationDomain" xlink:type="locator" xlink:label="us-gaap_IncomeStatementLocationDomain" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncomeStatementLocationDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Income Statement Location</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_IncomeStatementLocationDomain" xlink:to="us-gaap_IncomeStatementLocationDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_IncomeStatementLocationDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Income Statement Location [Domain]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_InterestIncomeMember" xlink:type="locator" xlink:label="us-gaap_InterestIncomeMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestIncomeMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Interest Income [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_InterestIncomeMember" xlink:to="us-gaap_InterestIncomeMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_InterestIncomeMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Interest Income [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ResidentialMortgageOrConsumerLoansMember" xlink:type="locator" xlink:label="pbhc_ResidentialMortgageOrConsumerLoansMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ResidentialMortgageOrConsumerLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Residential mortgage or consumer loans.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ResidentialMortgageOrConsumerLoansMember" xlink:to="pbhc_ResidentialMortgageOrConsumerLoansMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ResidentialMortgageOrConsumerLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Residential Mortgage or Consumer Loans [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ResidentialMortgageOrConsumerLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Residential Mortgage Or Consumer Loans [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ResidentialAndConsumerLoansMember" xlink:type="locator" xlink:label="pbhc_ResidentialAndConsumerLoansMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ResidentialAndConsumerLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Residential and consumer loans.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ResidentialAndConsumerLoansMember" xlink:to="pbhc_ResidentialAndConsumerLoansMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ResidentialAndConsumerLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Residential and Consumer Loans [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ResidentialAndConsumerLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Residential And Consumer Loans [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_CommercialRealEstateOrOtherCommercialAndIndustrialLoansMember" xlink:type="locator" xlink:label="pbhc_CommercialRealEstateOrOtherCommercialAndIndustrialLoansMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_CommercialRealEstateOrOtherCommercialAndIndustrialLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Commercial real estate or other commercial and industrial loans.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_CommercialRealEstateOrOtherCommercialAndIndustrialLoansMember" xlink:to="pbhc_CommercialRealEstateOrOtherCommercialAndIndustrialLoansMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_CommercialRealEstateOrOtherCommercialAndIndustrialLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Commercial Real Estate or Other Commercial and Industrial Loans [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_CommercialRealEstateOrOtherCommercialAndIndustrialLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Commercial Real Estate Or Other Commercial And Industrial Loans [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_NinetyDaysDeferralMember" xlink:type="locator" xlink:label="pbhc_NinetyDaysDeferralMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NinetyDaysDeferralMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">90 days deferral member.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_NinetyDaysDeferralMember" xlink:to="pbhc_NinetyDaysDeferralMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NinetyDaysDeferralMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Ninety Days Deferral [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NinetyDaysDeferralMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Ninety Days Deferral [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_NonDeferralMember" xlink:type="locator" xlink:label="pbhc_NonDeferralMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NonDeferralMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Non deferral.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_NonDeferralMember" xlink:to="pbhc_NonDeferralMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NonDeferralMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Non-Deferral [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NonDeferralMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Non Deferral [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_DeferralMember" xlink:type="locator" xlink:label="pbhc_DeferralMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DeferralMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Deferral</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_DeferralMember" xlink:to="pbhc_DeferralMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DeferralMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Deferral [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DeferralMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deferral [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_OneEightyDaysAndBeyondDeferralMember" xlink:type="locator" xlink:label="pbhc_OneEightyDaysAndBeyondDeferralMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_OneEightyDaysAndBeyondDeferralMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">180 days and beyond deferral.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_OneEightyDaysAndBeyondDeferralMember" xlink:to="pbhc_OneEightyDaysAndBeyondDeferralMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_OneEightyDaysAndBeyondDeferralMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">One Eighty Days and Beyond Deferral [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_OneEightyDaysAndBeyondDeferralMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">One Eighty Days And Beyond Deferral [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_IndividualLoansMember" xlink:type="locator" xlink:label="pbhc_IndividualLoansMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_IndividualLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Individual loans.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_IndividualLoansMember" xlink:to="pbhc_IndividualLoansMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_IndividualLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Individual Loans [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_IndividualLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Individual Loans [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_UncollateralizedMember" xlink:type="locator" xlink:label="us-gaap_UncollateralizedMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_UncollateralizedMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Unsecured Loans [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_UncollateralizedMember" xlink:to="us-gaap_UncollateralizedMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_UncollateralizedMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Uncollateralized [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_UnusualOrInfrequentItemLineItems" xlink:type="locator" xlink:label="us-gaap_UnusualOrInfrequentItemLineItems" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_UnusualOrInfrequentItemLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Unusual Or Infrequent Item [Line Items]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_UnusualOrInfrequentItemLineItems" xlink:to="us-gaap_UnusualOrInfrequentItemLineItems_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_UnusualOrInfrequentItemLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Unusual Or Infrequent Item [Line Items]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_PercentageOfDecreaseInFederalFundsRateTarget_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Percentage of decrease in federal funds rate target</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_PercentageOfDecreaseInFederalFundsRateTarget_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Percentage Of Decrease In Federal Funds Rate Target</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_BasisPointReductionInTargetFederalFundsRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Basis point reduction in target federal funds rate</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_BasisPointReductionInTargetFederalFundsRate_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Basis Point Reduction In Target Federal Funds Rate</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_LoansPayableContractualMaturitiesTermMaximum" xlink:type="locator" xlink:label="pbhc_LoansPayableContractualMaturitiesTermMaximum" />
    <link:label xml:lang="en-US" xlink:label="pbhc_LoansPayableContractualMaturitiesTermMaximum_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Loans payable contractual maturities term maximum.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_LoansPayableContractualMaturitiesTermMaximum" xlink:to="pbhc_LoansPayableContractualMaturitiesTermMaximum_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_NumberOfLoans" xlink:type="locator" xlink:label="pbhc_NumberOfLoans" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfLoans_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Number of loans.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_NumberOfLoans" xlink:to="pbhc_NumberOfLoans_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_LoansApproval" xlink:type="locator" xlink:label="pbhc_LoansApproval" />
    <link:label xml:lang="en-US" xlink:label="pbhc_LoansApproval_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Loans approval</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_LoansApproval" xlink:to="pbhc_LoansApproval_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_InterestOnLoan" xlink:type="locator" xlink:label="pbhc_InterestOnLoan" />
    <link:label xml:lang="en-US" xlink:label="pbhc_InterestOnLoan_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Interest on loan</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_InterestOnLoan" xlink:to="pbhc_InterestOnLoan_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_LoanOriginationFees" xlink:type="locator" xlink:label="pbhc_LoanOriginationFees" />
    <link:label xml:lang="en-US" xlink:label="pbhc_LoanOriginationFees_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Loan origination fees.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_LoanOriginationFees" xlink:to="pbhc_LoanOriginationFees_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_DeferredFeeIncome" xlink:type="locator" xlink:label="pbhc_DeferredFeeIncome" />
    <link:label xml:lang="en-US" xlink:label="pbhc_DeferredFeeIncome_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Deferred fee income.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_DeferredFeeIncome" xlink:to="pbhc_DeferredFeeIncome_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_InterestOnLoan_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">InterestOnLoan</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_InterestOnLoan_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Interest On Loan</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_LoansPayableContractualMaturitiesTermMaximum_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Loans Payable Contractual Maturities Term</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_LoansPayableContractualMaturitiesTermMaximum_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Loans Payable Contractual Maturities Term Maximum</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfLoans_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Number Of Loans</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfLoans_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Number Of Loans</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_LoansApproval_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Loans and Leases Receivable, Loans in Process</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_LoansApproval_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Loans Approval</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_LoanOriginationFees_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Loan origination fees</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_LoanOriginationFees_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Loan Origination Fees</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DeferredFeeIncome_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Deferred fee income</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_DeferredFeeIncome_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Deferred Fee Income</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_PercentageOfOutstandingLoans" xlink:type="locator" xlink:label="pbhc_PercentageOfOutstandingLoans" />
    <link:label xml:lang="en-US" xlink:label="pbhc_PercentageOfOutstandingLoans_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Percentage of outstanding loans.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_PercentageOfOutstandingLoans" xlink:to="pbhc_PercentageOfOutstandingLoans_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfLoansOutstanding_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Number of loans outstanding</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_PercentageOfOutstandingLoans_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Loan outstanding</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_PercentageOfOutstandingLoans_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Percentage Of Outstanding Loans</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_NumberOfRelatedCreditRelationships" xlink:type="locator" xlink:label="pbhc_NumberOfRelatedCreditRelationships" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfRelatedCreditRelationships_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Number of related credit relationships.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_NumberOfRelatedCreditRelationships" xlink:to="pbhc_NumberOfRelatedCreditRelationships_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_NumberOfIndividualLoans" xlink:type="locator" xlink:label="pbhc_NumberOfIndividualLoans" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfIndividualLoans_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Number of individual loans.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_NumberOfIndividualLoans" xlink:to="pbhc_NumberOfIndividualLoans_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfRelatedCreditRelationships_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Number of related credit relationships</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfRelatedCreditRelationships_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Number Of Related Credit Relationships</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfIndividualLoans_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Number of individual loans</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NumberOfIndividualLoans_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Number Of Individual Loans</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_NotesReceivableNet_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/positiveLabel">Loans</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_LoansAndLeasesReceivableAverageLoanBalance" xlink:type="locator" xlink:label="pbhc_LoansAndLeasesReceivableAverageLoanBalance" />
    <link:label xml:lang="en-US" xlink:label="pbhc_LoansAndLeasesReceivableAverageLoanBalance_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Loans and leases receivable, average loan balance.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_LoansAndLeasesReceivableAverageLoanBalance" xlink:to="pbhc_LoansAndLeasesReceivableAverageLoanBalance_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_LoansAndLeasesReceivablePercentOfLoans" xlink:type="locator" xlink:label="pbhc_LoansAndLeasesReceivablePercentOfLoans" />
    <link:label xml:lang="en-US" xlink:label="pbhc_LoansAndLeasesReceivablePercentOfLoans_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Loans and leases receivable percent of loans.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_LoansAndLeasesReceivablePercentOfLoans" xlink:to="pbhc_LoansAndLeasesReceivablePercentOfLoans_lbl" />
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_CommercialAndIndustrialLoansMember" xlink:type="locator" xlink:label="pbhc_CommercialAndIndustrialLoansMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_CommercialAndIndustrialLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Commercial and industrial loans.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_CommercialAndIndustrialLoansMember" xlink:to="pbhc_CommercialAndIndustrialLoansMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_CommercialAndIndustrialLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Commercial and Industrial Loans [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_CommercialAndIndustrialLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Commercial And Industrial Loans [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_SecuredTermLoansMember" xlink:type="locator" xlink:label="pbhc_SecuredTermLoansMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_SecuredTermLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Secured term loans.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_SecuredTermLoansMember" xlink:to="pbhc_SecuredTermLoansMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_SecuredTermLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Secured Term Loans [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_SecuredTermLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Secured Term Loans [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_UnsecuredTermLoansMember" xlink:type="locator" xlink:label="pbhc_UnsecuredTermLoansMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_UnsecuredTermLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Unsecured term loans.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_UnsecuredTermLoansMember" xlink:to="pbhc_UnsecuredTermLoansMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_UnsecuredTermLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Unsecured Term Loans [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_UnsecuredTermLoansMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Unsecured Term Loans [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_HomeEquityLoanMember" xlink:type="locator" xlink:label="us-gaap_HomeEquityLoanMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HomeEquityLoanMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Home Equity Lines of Credit [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_HomeEquityLoanMember" xlink:to="us-gaap_HomeEquityLoanMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_HomeEquityLoanMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Home Equity Loan [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_AllOtherMember" xlink:type="locator" xlink:label="pbhc_AllOtherMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_AllOtherMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">All other.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_AllOtherMember" xlink:to="pbhc_AllOtherMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_AllOtherMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">All Others [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AllOtherMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">All Other [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_CollateralAxis" xlink:type="locator" xlink:label="us-gaap_CollateralAxis" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CollateralAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Collateral Held</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_CollateralAxis" xlink:to="us-gaap_CollateralAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CollateralAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Collateral [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_CollateralDomain" xlink:type="locator" xlink:label="us-gaap_CollateralDomain" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CollateralDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Collateral Held</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_CollateralDomain" xlink:to="us-gaap_CollateralDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CollateralDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Collateral [Domain]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_CollateralPledgedMember" xlink:type="locator" xlink:label="us-gaap_CollateralPledgedMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CollateralPledgedMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Secured Loan [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_CollateralPledgedMember" xlink:to="us-gaap_CollateralPledgedMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CollateralPledgedMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Collateral Pledged [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/srt/2020/elts/srt-2020-01-31.xsd#srt_MortgageLoansOnRealEstateDescriptionTypeOfPropertyAxis" xlink:type="locator" xlink:label="srt_MortgageLoansOnRealEstateDescriptionTypeOfPropertyAxis" />
    <link:label xml:lang="en-US" xlink:label="srt_MortgageLoansOnRealEstateDescriptionTypeOfPropertyAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Real Estate, Type of Property</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="srt_MortgageLoansOnRealEstateDescriptionTypeOfPropertyAxis" xlink:to="srt_MortgageLoansOnRealEstateDescriptionTypeOfPropertyAxis_lbl" />
    <link:label xml:lang="en-US" xlink:label="srt_MortgageLoansOnRealEstateDescriptionTypeOfPropertyAxis_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Mortgage Loans On Real Estate Description Type Of Property [Axis]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/srt/2020/elts/srt-2020-01-31.xsd#srt_MortgageLoansOnRealEstateNamePropertyTypeDomain" xlink:type="locator" xlink:label="srt_MortgageLoansOnRealEstateNamePropertyTypeDomain" />
    <link:label xml:lang="en-US" xlink:label="srt_MortgageLoansOnRealEstateNamePropertyTypeDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Real Estate</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="srt_MortgageLoansOnRealEstateNamePropertyTypeDomain" xlink:to="srt_MortgageLoansOnRealEstateNamePropertyTypeDomain_lbl" />
    <link:label xml:lang="en-US" xlink:label="srt_MortgageLoansOnRealEstateNamePropertyTypeDomain_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Mortgage Loans On Real Estate Name Property Type [Domain]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_MixedUseMember" xlink:type="locator" xlink:label="pbhc_MixedUseMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_MixedUseMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Mixed use.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_MixedUseMember" xlink:to="pbhc_MixedUseMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_MixedUseMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Mixed Use [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_MixedUseMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Mixed Use [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/srt/2020/elts/srt-2020-01-31.xsd#srt_MultifamilyMember" xlink:type="locator" xlink:label="srt_MultifamilyMember" />
    <link:label xml:lang="en-US" xlink:label="srt_MultifamilyMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Multi-Family Residential [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="srt_MultifamilyMember" xlink:to="srt_MultifamilyMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="srt_MultifamilyMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Multifamily [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/srt/2020/elts/srt-2020-01-31.xsd#srt_HotelMember" xlink:type="locator" xlink:label="srt_HotelMember" />
    <link:label xml:lang="en-US" xlink:label="srt_HotelMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Hotels and Motels [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="srt_HotelMember" xlink:to="srt_HotelMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="srt_HotelMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Hotel [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/srt/2020/elts/srt-2020-01-31.xsd#srt_OfficeBuildingMember" xlink:type="locator" xlink:label="srt_OfficeBuildingMember" />
    <link:label xml:lang="en-US" xlink:label="srt_OfficeBuildingMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Office [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="srt_OfficeBuildingMember" xlink:to="srt_OfficeBuildingMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="srt_OfficeBuildingMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Office Building [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/srt/2020/elts/srt-2020-01-31.xsd#srt_RetailSiteMember" xlink:type="locator" xlink:label="srt_RetailSiteMember" />
    <link:label xml:lang="en-US" xlink:label="srt_RetailSiteMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Retail [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="srt_RetailSiteMember" xlink:to="srt_RetailSiteMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="srt_RetailSiteMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Retail Site [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/srt/2020/elts/srt-2020-01-31.xsd#srt_SingleFamilyMember" xlink:type="locator" xlink:label="srt_SingleFamilyMember" />
    <link:label xml:lang="en-US" xlink:label="srt_SingleFamilyMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">1-4 Family Residential [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="srt_SingleFamilyMember" xlink:to="srt_SingleFamilyMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="srt_SingleFamilyMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Single Family [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_AutomobileDealershipMember" xlink:type="locator" xlink:label="pbhc_AutomobileDealershipMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_AutomobileDealershipMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Automobile dealership.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_AutomobileDealershipMember" xlink:to="pbhc_AutomobileDealershipMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_AutomobileDealershipMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Automobile Dealership [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AutomobileDealershipMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Automobile Dealership [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/srt/2020/elts/srt-2020-01-31.xsd#srt_WarehouseMember" xlink:type="locator" xlink:label="srt_WarehouseMember" />
    <link:label xml:lang="en-US" xlink:label="srt_WarehouseMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Warehouse [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="srt_WarehouseMember" xlink:to="srt_WarehouseMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="srt_WarehouseMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Warehouse [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_RecreationOrGolfCourseOrMarinaMember" xlink:type="locator" xlink:label="pbhc_RecreationOrGolfCourseOrMarinaMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_RecreationOrGolfCourseOrMarinaMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Recreation or golf course or marina.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_RecreationOrGolfCourseOrMarinaMember" xlink:to="pbhc_RecreationOrGolfCourseOrMarinaMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_RecreationOrGolfCourseOrMarinaMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Recreation/Golf Course/Marina [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_RecreationOrGolfCourseOrMarinaMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Recreation Or Golf Course Or Marina [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_SkilledNursingFacilityMember" xlink:type="locator" xlink:label="pbhc_SkilledNursingFacilityMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_SkilledNursingFacilityMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Skilled nursing facility.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_SkilledNursingFacilityMember" xlink:to="pbhc_SkilledNursingFacilityMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_SkilledNursingFacilityMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Skilled Nursing Facility [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_SkilledNursingFacilityMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Skilled Nursing Facility [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_ManufacturingOrIndustrialMember" xlink:type="locator" xlink:label="pbhc_ManufacturingOrIndustrialMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ManufacturingOrIndustrialMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Manufacturing or industrial.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_ManufacturingOrIndustrialMember" xlink:to="pbhc_ManufacturingOrIndustrialMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_ManufacturingOrIndustrialMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Manufacturing/Industrial [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_ManufacturingOrIndustrialMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Manufacturing Or Industrial [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_RestaurantMember" xlink:type="locator" xlink:label="pbhc_RestaurantMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_RestaurantMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Restaurant.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_RestaurantMember" xlink:to="pbhc_RestaurantMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_RestaurantMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Restaurant [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_RestaurantMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Restaurant [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_AutomobileRepairMember" xlink:type="locator" xlink:label="pbhc_AutomobileRepairMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_AutomobileRepairMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Automobile repair.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_AutomobileRepairMember" xlink:to="pbhc_AutomobileRepairMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_AutomobileRepairMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Automobile Repair [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_AutomobileRepairMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Automobile Repair [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_NotForProfitAndCommunityServiceMember" xlink:type="locator" xlink:label="pbhc_NotForProfitAndCommunityServiceMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NotForProfitAndCommunityServiceMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Not for profit and community service real estate.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_NotForProfitAndCommunityServiceMember" xlink:to="pbhc_NotForProfitAndCommunityServiceMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_NotForProfitAndCommunityServiceMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Not-For-Profit &amp; Community Service Real Estate [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_NotForProfitAndCommunityServiceMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Not For Profit And Community Service [Member]</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_HospitalsMember" xlink:type="locator" xlink:label="pbhc_HospitalsMember" />
    <link:label xml:lang="en-US" xlink:label="pbhc_HospitalsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Hospitals member.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_HospitalsMember" xlink:to="pbhc_HospitalsMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="pbhc_HospitalsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Hospitals [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_HospitalsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Hospitals [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/srt/2020/elts/srt-2020-01-31.xsd#srt_OtherPropertyMember" xlink:type="locator" xlink:label="srt_OtherPropertyMember" />
    <link:label xml:lang="en-US" xlink:label="srt_OtherPropertyMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">All Other [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="srt_OtherPropertyMember" xlink:to="srt_OtherPropertyMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="srt_OtherPropertyMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Other Property [Member]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_CollateralizedLoanObligationsMember" xlink:type="locator" xlink:label="us-gaap_CollateralizedLoanObligationsMember" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CollateralizedLoanObligationsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Collateral [Member]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_CollateralizedLoanObligationsMember" xlink:to="us-gaap_CollateralizedLoanObligationsMember_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_CollateralizedLoanObligationsMember_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Collateralized Loan Obligations [Member]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_LoansAndLeasesReceivableAverageLoanBalance_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Average Loan Balance</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_LoansAndLeasesReceivableAverageLoanBalance_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Loans And Leases Receivable Average Loan Balance</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_LoansAndLeasesReceivableAllowance_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/verboseLabel">Allowance for Loan Losses</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_LoansAndLeasesReceivablePercentOfLoans_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Percent of Total Loans</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_LoansAndLeasesReceivablePercentOfLoans_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Loans And Leases Receivable Percent Of Loans</link:label>
    <link:label xml:lang="en-US" xlink:label="us-gaap_LoansAndLeasesReceivableDeferredIncome_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Net deferred loan fees</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_LoansAndLeasesReceivableBeforeFeesGross" xlink:type="locator" xlink:label="us-gaap_LoansAndLeasesReceivableBeforeFeesGross" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LoansAndLeasesReceivableBeforeFeesGross_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Total Loans</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_LoansAndLeasesReceivableBeforeFeesGross" xlink:to="us-gaap_LoansAndLeasesReceivableBeforeFeesGross_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_LoansAndLeasesReceivableBeforeFeesGross_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Loans And Leases Receivable Before Fees Gross</link:label>
    <link:loc xlink:href="pbhc-20201231.xsd#pbhc_WarrantStockDividendsPerShareDeclared" xlink:type="locator" xlink:label="pbhc_WarrantStockDividendsPerShareDeclared" />
    <link:label xml:lang="en-US" xlink:label="pbhc_WarrantStockDividendsPerShareDeclared_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/documentation">Warrant stock dividends per share declared.</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="pbhc_WarrantStockDividendsPerShareDeclared" xlink:to="pbhc_WarrantStockDividendsPerShareDeclared_lbl" />
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_SubsequentEventTable" xlink:type="locator" xlink:label="us-gaap_SubsequentEventTable" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SubsequentEventTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Subsequent Event [Table]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_SubsequentEventTable" xlink:to="us-gaap_SubsequentEventTable_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SubsequentEventTable_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Subsequent Event [Table]</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_SubsequentEventLineItems" xlink:type="locator" xlink:label="us-gaap_SubsequentEventLineItems" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SubsequentEventLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Subsequent Event [Line Items]</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_SubsequentEventLineItems" xlink:to="us-gaap_SubsequentEventLineItems_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_SubsequentEventLineItems_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Subsequent Event [Line Items]</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_WarrantStockDividendsPerShareDeclared_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Cash dividend declared on warrant</link:label>
    <link:label xml:lang="en-US" xlink:label="pbhc_WarrantStockDividendsPerShareDeclared_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Warrant Stock Dividends Per Share Declared</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DividendPayableDateToBePaidDayMonthAndYear" xlink:type="locator" xlink:label="us-gaap_DividendPayableDateToBePaidDayMonthAndYear" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DividendPayableDateToBePaidDayMonthAndYear_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Cash dividend payable date</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DividendPayableDateToBePaidDayMonthAndYear" xlink:to="us-gaap_DividendPayableDateToBePaidDayMonthAndYear_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DividendPayableDateToBePaidDayMonthAndYear_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Dividend Payable Date To Be Paid Day Month And Year</link:label>
    <link:loc xlink:href="http://xbrl.fasb.org/us-gaap/2020/elts/us-gaap-2020-01-31.xsd#us-gaap_DividendsPayableDateOfRecordDayMonthAndYear" xlink:type="locator" xlink:label="us-gaap_DividendsPayableDateOfRecordDayMonthAndYear" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DividendsPayableDateOfRecordDayMonthAndYear_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/terseLabel">Cash dividend record date</link:label>
    <link:labelArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="us-gaap_DividendsPayableDateOfRecordDayMonthAndYear" xlink:to="us-gaap_DividendsPayableDateOfRecordDayMonthAndYear_lbl" />
    <link:label xml:lang="en-US" xlink:label="us-gaap_DividendsPayableDateOfRecordDayMonthAndYear_lbl" xlink:type="resource" xlink:role="http://www.xbrl.org/2003/role/label">Dividends Payable Date Of Record Day Month And Year</link:label>
  </link:labelLink>
</link:linkbase>
