\documentclass[11pt]{article}
\input{headers01}

\lhead{CS 585-Theoretical CS Toolkit, SPRING 2023}               
\rhead{Name: Write your name} %%% <-- REPLACE Hemanta K. Maji WITH YOUR NAME HERE

\usepackage[strict]{changepage}  
\newcommand{\nextoddpage}{\checkoddpage\ifoddpage{\ \newpage\ \newpage}\else{\ \newpage}\fi}  


\begin{document}

\title{Homework 1}

\date{}

\maketitle 

\thispagestyle{fancy}  
\pagestyle{fancy}      


{\bfseries Collaborators :} \newline 
% ENTER THEIR NAMES HERE 

\begin{enumerate}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%% PROBLEM 1 %%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\item {\bfseries Upper-bound on Entropy.} (20 points)
Let $\Omega=\{1,2,\dotsc,N\}$. 
Suppose $\X$ is a random variable over the sample space $\Omega$. 
For shorthand, let $p_i = \probX{\X=i}$, for each $i\in\Omega$. 
The random variable $\ X$'s entropy is defined as the following function. 
  $$ \HH{\X} \defeq \sum_{i\in\Omega} -p_i\cdot\ln p_i.$$
Use Jensen's inequality on the function $f(t)=\ln t$ to prove the following inequality. 
  $$ \HH{\X} \leq \ln N.$$
Furthermore, equality holds if and only if $\X$ is the uniform distribution over $\Omega$. 

\noindent{\bfseries Solution.}\newline 
\   %%% <-- ERASE THIS LINE AND WRITE YOUR SOLUTION HERE
\newpage


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%% PROBLEM 2 %%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\item {\bfseries Log-sum Inequality.} (27=22+5 points)
    \begin{enumerate}
        \item Let $\{a_1,\dotsc,a_N\}$ and $\{b_1,\dotsc,b_N\}$ be two sets of positive real numbers. 
        Use Jensen's inequality to prove the following inequality. 
        $$\sum_{i=1}^N a_i \ln \frac{a_i}{b_i} \geq A\ln\frac AB,$$
        where $A \defeq \sum_{i=1}^Na_i$ and $B \defeq \sum_{i=1}^Nb_i$. 
        Furthermore, equality holds if and only if $a_i/b_i$ is identical for all $i\in\{1,\dotsc,N\}$.
        \newline
        \noindent{\bfseries Solution.}\newline 
\   %%% <-- ERASE THIS LINE AND WRITE YOUR SOLUTION HERE
        \newpage
        \item Let $\cX$ be a finite set and $P:\cX\to [0,1]$ and $Q:\cX\to [0,1]$ be two probability distributions on $\cX$ such that for any $x\in \cX$, $Q(x)\not=0$. The relative entropy from $Q$ to $P$ is defined as follows:
        $$\kl{P}{Q}\defeq \sum_{x\in \cX} P(x)\log{\frac{P(x)}{Q(x)}}.$$
        Show that for any $P$ and $Q$, it holds that $\kl{P}{Q}\geq 0$. Moreover, state when $\kl{P}{Q}=0$.
    \end{enumerate}
\noindent{\bfseries Solution.}\newline 
\   %%% <-- ERASE THIS LINE AND WRITE YOUR SOLUTION HERE
\newpage

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%% PROBLEM 3 %%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\item {\bfseries Approximating Square-root.} (20 points)
Our objective is to find a (meaningful and tight) lower bound for the function $f(x) = (1-x)^{-1/2}$ when $x\in [0,1)$ using a quadratic function of the form
  $$g(x) = 1 +\alpha x+\beta x^2.$$
Use the Lagrange form of Taylor's remainder theorem on $f(x)$ around $x=0$ to obtain the function $g(x)$. 

\noindent{\bfseries Solution.}\newline 
\   %%% <-- ERASE THIS LINE AND WRITE YOUR SOLUTION HERE
\newpage





%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%% PROBLEM 4 %%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\item {\bfseries Lower-bounding Logarithm Function.} (20 points)
By Taylor's Theorem, we have seen that the following upper bound is true. 
\begin{boxedalgo}
For all $\eps\in[0,1)$ and integer $k\geq 1$, we have 
  $$\ln(1-\eps) \leq -\eps - \frac{\eps^2}{2} - \dotsi - \frac{\eps^k}k$$
\end{boxedalgo}
We want a tight lower bound for $\ln(1-\eps)$. 
Prove the following lower-bound.
\begin{boxedalgo}
For all $\eps\in[0,1/2]$ and integer $k\geq 1$, we have 
  $$\ln(1-\eps) \geq \left(-\eps - \frac{\eps^2}{2} - \dotsi - \frac{\eps^k}k\right) - \frac{\eps^k}{k}$$
\end{boxedalgo}
(For visualization of this bound, follow this \href{https://www.desmos.com/calculator/o3iwil80fp}{link})

\noindent{\bfseries Solution.}\newline 
\   %%% <-- ERASE THIS LINE AND WRITE YOUR SOLUTION HERE
\newpage


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%% PROBLEM 5 %%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\item {\bfseries Using Stirling Approximation.} (23 points)
Suppose we have a coin that outputs heads with probability $p$ and outputs tails with probability $q=1-p$. 
We toss this coin (independently) $n$ times and record each outcome. 
Let $\H$ be the random variable representing the number of heads in this experiment. 
Note that the following expression gives the probability that we get a total of $k$ heads. 
  $$\probX{\H=k} = \choose n k p^k q^{n-k}$$
We will prove upper and lower bounds for this problem, assuming $k \geq pn$. 
Define $p'\defeq k/n=(p+\eps)$. 

Let $P$ and $P'$ be two probability distributions on the set $\cX=\{\text{tails},\text{heads}\}$ such that
 $\pr(P=\text{heads})=p$ and $\pr(P'=\text{heads})=p'$.
 
Using the (Robbin's form of) Stirling approximation in the lecture notes, prove the following bound.  
  $$ \frac1{\sqrt{8n p'(1-p')}}\exp\left(-n\kl{P'}{P}\right) \leq \probX{\H=k} \leq \frac1{\sqrt{2\pi n p'(1-p')}}\exp\left(-n\kl{P'}{P}\right),$$
where $\kl{P'}{P}$ is the relative entropy from $P$ to $P'$ defined in question 3.

\noindent{\bfseries Solution.}\newline 
\   %%% <-- ERASE THIS LINE AND WRITE YOUR SOLUTION HERE
\newpage
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%% PROBLEM 6 %%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\item \textbf{Computing a limit.} (20 points) Compute the following limit
 $$\lim_{n\to \infty} \sum_{j=1}^n\frac{\sqrt{4n^2-j^2}}{n^2}.$$

\noindent{\bfseries Solution.}\newline 
\   %%% <-- ERASE THIS LINE AND WRITE YOUR SOLUTION HERE
 \newpage
%  %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% %%%%%%%%%%%% PROBLEM 7 %%%%%%%%%%%%%%%%%%%%%%%%%%%%
% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     
 \newpage   


\item \textbf{Birthday Bound.} (20 points) 
    Intuitively, we want to claim that the following two expressions are ``good approximations'' of each other. 
        $$ f_n(t) \defeq \left(1-\frac1n\right) \left(1-\frac2n\right) \dotsi \left( 1-\frac{t-1}{n}\right)$$
    And
        $$ g_n(t) \defeq \exp\left(-\frac{t^2}{2n}\right)$$
    To formalize this intuition, write the mathematical theorems (and then prove them) when $t=o(n^{2/3})$.

    Hint: You may find the following inequalities helpful.  
    \begin{enumerate}
    \item  $\ln(1-x)\leq -x$, for $x\in[0,1)$, and 
    \item $\ln(1-x) \geq -x-x^2$, for $x\in[0,1/2]$ (you already prove this identity earlier). 
    \end{enumerate}
  \newpage

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%% PROBLEM 8 %%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\item {\bfseries Tight Estimation: Central Binomial Coefficient.} (Extra credit: 15 points)
    We will learn a new powerful technique to prove tight inequalities. 
    As a representative example, we will estimate the central binomial coefficient. 
    For positive integer $n$, we will prove that 
        $$L_n \leq \binom{2n}{n} \leq U_n ,$$
    where 
        \begin{align*}
            L_n &\defeq \frac{4^n}{\sqrt{\pi\left(n + \frac14 + \frac1{32n}\right)}} &&&
            U_n &\defeq \frac{4^n}{\sqrt{\pi\left(n + \frac14 + \frac1{46n}\right)}}.
        \end{align*}

    To prove these bounds, we will use the following general strategy. 
    \begin{enumerate}
    \item Define the following two sequences 
        \begin{align*}
            \left\{a_n \defeq \binom{2n}{n}/U_n \right\}_{n} &&& \left\{b_n \defeq \binom{2n}{n}/L_n \right\}_{n}
        \end{align*}
    \item {\em Prove the following limit.}
            $$ \lim_{n\to\infty} a_n = \lim_{n\to\infty} b_n = \lim_{n\to\infty} \frac{\binom{2n}{n}}{4^n/\sqrt{\pi n}} = 1, $$
        using the Stirling approximation $n! \sim \sqrt{2\pi n}\cdot(n/\mathrm e)^n$. 
    \item {\em Prove $\left\{a_n\right\}_n$ is an increasing sequence.}
    \item {\em From (b) and (c), conclude that $a_n\leq 1$, implying $\binom{2n}{n} \leq U_n$.}
    \item {\em Prove $\left\{b_n\right\}_n$ is a decreasing sequence.}
    \item {\em From (b) and (e), conclude that $b_n\geq 1$, implying $\binom{2n}{n} \geq L_n$.}
    \end{enumerate} 


    \begin{boxedalgo}
    {\em Remark: What did we achieve from this exercise? }
    We started from the asymptotic estimate $\binom{2n}{n} \sim 4^n/\sqrt{\pi n}$. 
    From this asymptotic estimate, we obtained explicit upper and lower bounds. 
    We learned a powerful general technique to translate asymptotic estimates into explicit upper and lower bounds automatically. 
    \end{boxedalgo}
  
  {\bfseries Solution.} \newline 
  
   \   %%% <-- ERASE THIS LINE AND WRITE YOUR SOLUTION HERE

\end{enumerate}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%% PLEASE LIST COLLABORATORS BELOW  %%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 

\end{document}
