elad.cs.technion.ac.il · SD ! !( PARTAN " ... mn m n N mn n m M n n N m m M ... 02 fx x 0 fx x xx fx x xx fx x xx fx x

Linear and Non-Linear Programming David G. Luenberger

!!" !!#

Simplex

$

1. David G. Luenberger, Linear and Non Linear Programming, Addison-Wesley Pub.

Co. , Menlo-Park, California, Second Edition, 1987.

2. Dimitri P. Bertsekas, Nonlinear Programming, Athena Scientific, Belmont,

Massachusetts, 1995.

3. W. H. Press, S. A. Teukolsky, W. T. Vatterling & B. P. Flannery, Numerical

recepies In C, Cambridge University Press, 1987.

4. R. Fletcher, Practical Methods of Optimization, Jhon Wiley & Sons, Second

Edition, 1987.

5. R. Meir, Lecture Notes in Optimization Theory

6. L.A. Hageman & D. M. Young, Applied Iterative Methods, Academic Press, 1981.

7. Donald A. Pierre, Optimization Theory with Applications, Dover Publications,

Inc., New York, 1986.

%&

&"

'

( &

% (

%% )

%& !

%'%%

%(%'

%)%"

& &

&%&&

&&&(

&'&"

' &!

'%'

'&'%

'''&

'(')

')(*

+ ( (

, (%('

(&)*

(')&

(()(

())#

(#)#

) )!

Simplex )%)!

)&#&

)'#'

# #(

#%##

SD #&#!

NSD #'"'

#("#

#)"!

DNSD ##!%

#"!'

SD !

PARTAN " !(

"%!"

SD "& *%

"' *)

CG "

! *#

CG !% %

CG !& (

CG !' %%

CG !( %'

!) %(

"

* %#

*% %#

DFP *& &

Broyden *' &)

*( &!

*) '%

*# '&

# LS

'(

LS % '#

Gauss-Newton LS & '!

LS ' ('

LS ( (#

) ))

#

% )#

%% #*

%& #&

Simplex %' #"

Simplex %( "#

Simplex %) !*

%# !

%" !'

& !(

&% !)

&&%*

, &'%*"

&(% %

&)% #

# $

' % !

The Penalty Method '%% !

The Berrier Method '&%%'

''%%#

Interior Point '(%&

, ')%&&

'#%&)

x n ( )f x n: R R→ 1

x

( )x ArgMin f x= f

!

" x

#

x ∈Ω

( )h x = 0

( )g x = 0

$

! !

#

"

%

"

! !

!

"

#

( )k n h x x xk n= =1 01 2,2, , , , ,

#

( ) ( )f x x x h x x xn k nk

n1 2

21 2

1, , , , , , = ∑

=

( )f x x xn1 2, , ,

" "

!!

&

"

!

"

M

! n m N

Sm m Cn n Pm n,

'

Xm n,

n m

X S X C

C S m,n

m n mn

Nm n n

m

M

nn

Nm

m

Mm n

, ,

,

=∑ =∑

⇒ ∑ = ∑ ≥ ∀

= =

= =

1 1

1 10

m = 1,2, ,M n = 1,2, ,N

X

N M

(

# !

Cost X Pm n m nn

N

m

M= ∑∑

==, ,

11

"

Simplex

10 50 100 500, , , n n n n1 2 3 4, , , $

m m m m1 2 3 4, , ,

! )* $

+ nn nn nn nn1 2 3 4, , ,

#

0 4

10 50 100 500 8501 2 3 4

≤ ≤ ≤ ≤+ + + =

nn n k

nn nn nn nnk k 1

( (

*

*

Weight m nnk kk

= ⋅∑=1

4

"

"

X

"

Yk Y H X Nk k k= +

Nk k Hk k

X M k

#

( ) [ ] [ ]ε2 2

1 1X Y H X Y H X Y H Xk k

k

Mk k

Tk k

k

M= −∑ = − −∑

= =

X

"

x

m x0

k ,

" x x uk k k+ = +1 #! uk

#

( )ε α β21 2

2

0

2

0u u u x um k

k

mk

k

m, , , = ⋅ ∑ + ⋅ ∑

= =

x x u x u u x uk k k k k k jj

k= + = + + = = + ∑− − − − −

=

−1 1 2 2 1 0

0

1

( )ε α β20 1 1 0

0

12

0

1 2

0

1u u u x u um j

j

k

k

mk

k

m, , , −

=

−

=

−

=

−= ⋅ + ∑

∑ + ⋅ ∑

!

!

xk kN

=1 AR

! I

! uk kN

=1 !

x a x uk k k= ⋅ +−1

" uk kN

=1 a

E x a x x a x x a xk kk

Nk k

k

N21

2

11 0

21

2

2= − ⋅∑ = − ⋅ + − ⋅∑−

=−

=

#

a x, 0

! xk kn

=1 n

y c w xk kk

n= + ∑

=φ

1

φ wk c

Multilayered ( )φ x x= sinh( )

Perceptron

$ $

!

P

X y y=h(P,X)

!

$%%

N ABC

P1

P2

P3

P4

P5

P6

P7X y

&

X ykk k

N,

=1

( )[ ]ε2 2

1P y h P Xk

k

k

N= −∑

=,

P

!

LS

$ '

! (

!

n m( )≥ Vk km

=1 m

α k km

=1

αk kk

mV =∑

=0

1

3

)

! x y x yi ik

n T

=∑ =

1 y x

x

x y x yT < ⋅ *

A

m ai j, j i

n

I $

!

!

ci j, B A C=AB

B j A i

c a bi j i k k jk

n, , ,= ∑

=1

j,i i,j Transpose (!

(! (! AT

! ! ! AH

det(A) ! A A A= T

!

!

! !

.A A AA I− −= =1 1 A−1

! B A

U B SAS= −1 S

$%

U U− =1 H n !

B UAU= H U B A

Ax x= λ x λ

det Ax x− =λ 0 !

!

Au u u uk k k k k kT

k

n= ⇒ = ∑

=λ λ A

1

! !

x A ! !

! x AxH

!

!

A Axx

==

∆max

1

A AT

$

Q I Q I Q I Qk

k

L LL=

+ −→∞

−∑ = − − → −0

1 1 1

m ,m Rn ! M !

M !

M ! M ⊥

x= x xn1 f(x)

x ( ) ( )f x f xk → x xk →

( ) ( )f x f y− < ε x y− < δ y δ > 0 ε > 0

Weierstrass

x p ( )f x Cp∈

f(x)∈ C1

x

( ) ( ) ( )∇ =

f x

f x

x

f x

xn

∂∂

∂∂1

( )f x C∈ 2

( ) ( )

( ) ( )

( ) ( )∇ = =

2

2

1 1

2

1

2

1

2f x x

f x

x x

f x

x x

f x

x x

f x

x x

n

n n n

F

∂∂ ∂

∂∂ ∂

∂∂ ∂

∂∂ ∂

( )f x C∈ 2

x d f(x)∈ C1

! α " ( )x x dα α= +

( ) ( )( ) ( )g f x f x dα α α= = +

d x f(x)

#

( ) ( ) ( ) ( )∂∂

αα

αα

∂∂

f x

x ddg

d

df x d

d

f x

xd= = + =

d

d ( )f x C∈ 2

( ) ( ) ( ) ( )d g

d

d f x d

d

d

d

f x d

xd d f x dT

2

2

2

22α

αα

α α∂ α

∂= + = + = ∇

y x f(x)∈ C1

( ) ( ) ( )( ) ( )( ) ( ) ( )( )( ) [ ]

f y f x f x y x o x y

f y f x f x y y x

= + ∇ − + −

= + ∇ + − − ∈θ θ θ1 0 1 ,

x y

! y x " x

Lim

0α

αα→

=o0

C2

( ) ( ) ( )( ) ( ) ( )( ) ( )( ) ( ) ( )( ) ( ) ( )( )( )

f y f x f x y x x y f x x y o x y

f y f x f x y x x y f x y x y

T

T

= + ∇ − + − ∇ − + −

= + ∇ − + − ∇ + − −

1

21

21

2 2

2 θ θ

$

N ε > 0 x xk k=∞

1

xk k=∞

1 x x xk − < ε k N≥

y x y − < ε x !sphere"

S ε x

S

!"

S

S

Weierstrass

%

x

x x ( )f x

n n

( )f x

x f Weierstras

Ω x

! !

" ! !

" x*

# ( ) ( )∃ε > ∀ − < ≥0| * * fx x x f xε

( ) ( )∀ ≥x f x f x* x*

#

!

!

!

$

d x

! ( )( ) ( )f x gα α= ( )x x dα α= +

f

%

f x* f C∈ 1

f

( )∇ ≡f x* 0

( )( ) ( )f x gα α= ( )x x dα α= +* d

" α = 0 !

( ) ( ) ( ) ( )g g g oα α α− = +0 0'

( )g' 0 0< α ! ( )o α α → 0

" ( ) ( )g gα − <0 0 α

&

( ) ( ) ( ) ( )gdg

d

df x d

df x d'

**0

0 00=

==

+

== ∇ ⋅ ≥α

α α

α

α α

( )∇ ≡f x* 0 d

n x*

( )∇ ≡f x* 0

( )f x x x x x x x1 2 12

1 2 22

23, = − + −

"

( )( )

( )∇ =

=−

− + −

=f x x

f x x

xf x x

x

x x

x x1 2

1 2

1

1 2

2

1 2

1 2

2

2 30,

,

,

∂∂

∂∂

x1 21 2= = x,

!

x ( )g x

( ) g g xk km

k km

= ==1 1 xk k

m=1 m

n<m ( )h x a xkk

k

n= ⋅∑

=0 n

'

( ) ( ) ( )[ ]ε20 1

2

1a a a h x g xn k k

k

m, , , = −∑

=

"

(

( )ε20 1

0

2

1a a a a x gn j

j

n

kj

kk

m, , , = ∑ −

∑==

" ai

( )∂∂

εa

a a a x a x gi

n ki

jj

n

kj

kk

m20 1

012 0, , , = ∑ −

∑ ===

" b x g x xi ki

kk

mki

kj

k

m

i j= ⋅∑ = ⋅∑

= =1 1 q;

,

q a bi j j ij

n, ⋅ −∑ =

=00

" ) n * i

q q q

q q q

q q q

a

a

a

b

b

b

n

w n

n n n n n n

0 0 0 0 0

1 0 11

0 1

0

1

0

1

, , ,

, , ,

, , ,

=

⇔ = Qa b

! !

x

m ! x0

k ! ! *

# x x uk k k+ = +1 " uk

"

( )ε α β21 2

2

0

2

0

1u u u x um k

k

mk

k

m, , , = ⋅ ∑ + ⋅ ∑

= =

−

x x u x u u x uk k k k k k jj

k= + = + + = = + ∑− − − − −

=

−1 1 2 2 1 0

0

1

( )ε α β20 1 1 0

0

12

0

1 2

0

1u u u x u um j

j

k

k

mk

k

m, , , −

=

−

=

−

=

−= ⋅ + ∑

∑ + ⋅ ∑

ui

[ ]

∂ε∂

α β

α α β

2

00

1

1

1

00

1

1

1

2 2 0

1 0

ux u u

m i x u u

ij

j

k

k i

mi

jj

k

k i

mi

= ⋅ + ∑

∑ + =

⇒ − − + ∑∑ + =

=

−

= +

−

=

−

= +

−

m m

!

! ( )f x

( )f x

x

( ) ( )

( ) ( )

( ) ( )F x f x

f x

x x

f x

x x

f x

x x

f x

x x

n

n n n

= ∇ =

2

2

1 1

2

1

2

1

2

∆

∂∂ ∂

∂∂ ∂

∂∂ ∂

∂∂ ∂

! A A>0

"#

x* $f C∈ 2

% f

( )∇ ≥2f x* 0 ( )∇ ≡f x* 0 f

d "

( )( ) ( )f x gα α= ( )x x dα α= +*

α = 0

( ) ( ) ( ) ( ) ( ) ( ) ( )g g g g o g oα α α α α α− = + + = +0 01

20

1

202 2 2 2' ' ' ' '

( )g' 0 α

( ) ( )g gα − <0 0 α ( )g' ' 0 0<

( ) ( )( )

( )( ) ( )g

d g

d

d f x d

dd f x dT' '

**0

0 00

2

2

2

22=

==

+

== ∇ ⋅ ≥α

α α

α

α α

( )∇ ≥2f x* 0 d

f

%

$

&

"

( )( )

( )

( )

( )∇ =

=−

−

21 2

21 2

1 12

1 2

2 1

21 2

1 22

1 2

2 2

2 1

1 2f x x

f x x

x x

f x x

x x

f x x

x x

f x x

x x

,

,

,

,

,

∂∂ ∂

∂∂ ∂

∂∂ ∂

∂∂ ∂

"

d

[ ] [ ]

[ ] ( )[ ]

d d d

d

d d d d d

d

d d d d d d d d

1 2 1

2

1 2 1 2 1

2

12

1 2 22

1 22

12

22

2 1

1 2

2 2

2 0

−−

=− − +

=

= − + = − + + ≥

x1 21 2= = x,

"

Q ! ""

Qa=b

X

X

x x x

x x x

x x x

n

n

m m mn

=

1

1

1

1 12

1

2 22

2

2

X Q Q X XT=

[ ]xk kn

=1 Vandemonde

Q m

( )f x x x x x x1 2 13

12

2 222, = − +

""

( )∇ = −− +

=f x xx x x

x x1 2

12

1 2

12

2

3 2

4, 0

x1 26 9= = x, x1 20 0= = x,

( )∇ =− −

−

⇒

= = ⇒ =

≥

= = ⇒ =−

−

21 2

1 2 1

1

1 2

1 2

6 2 2

2 4

0 00 0

0 40

6 918 12

12 4

f x xx x x

x

x F

x F

,

,

,

x

x

x1 20 0= = x,

x1 26 9= = x,

&$"% "'

%

$

&

x1 26 9= = x,

' (

$ %

x* $f C∈ 2

% f

( )∇ ≡f x* 0 x* f

f x* ( )∇ >2f x* 0

d β > 0 ( ) ( )F 0x f x* *= ∇ >2

β ( )d x d dTF * ≥ β 2

( )F x U UH* = ∆ F(x*) F

∆ U

( ) ( )

( )

d x d d U Ud d d d k

d k d d

T T H

Ud d

Hk

k

k

F *

min min min

= = = ⋅∑ ≥

≥ ∑ = =

↑=

∆ ∆

1

1

1

1 12

21

2 2

λ

λ λ λ

x* f

( ) ( ) ( ) ( ) ( )f x d f x f x d d x d o dT* * * *+ = + ∇ ⋅ + +1

22F

( )∇ >2f x* 0 ( )∇ ≡f x* 0 x*

!

"

( ) ( ) ( ) ( ) ( )f x d f x d x d o d d o dT* * *+ − = + ≥ ⋅ +1

2

1

22 2 2F β

x* d # d 2

2 1

1 2

−−

$ "

x1 21 2= = x,

0 0

0 4

x1 20 0= = x, $

#

x

x ∈Ω f(x)

% $

x ∈Ω f(x)

ε0 0> d x ∈Ω

( )x x dε ε= + ∈Ω 0 0≤ ≤ε ε

Ω

Ω &Ω '

(

)

x* &f C∈ 1

' f &% '

( )∇ ≥f x d* 0 d Ω f

d %

( )( ) ( )f x gε ε= &0 0≤ ≤ε ε ' ( )x x dε ε= +* ε0

# ε = 0

( ) ( ) ( ) ( )g g g oε ε ε− = +0 0'

( ) ( )g gε − <0 0 ε ( )g' 0 0<

( ) ( ) ( ) ( )gdg

d

df x d

df x d'

**0

0 00=

==

+

== ∇ ⋅ ≥ε

ε ε

ε

ε ε

Ω x

% ( )∇ ≡f x* 0 d #

x* &f C∈ 2

' f & '

( )∇ ≥f x d* 0 d Ω f

( )d f x dT∇ ≥2 0* ( )∇ =f x d* 0

d "

( )( ) ( )f x gε ε= &0 0≤ ≤ε ε ' ( )x x dε ε= +* ε0

# ε = 0

( ) ( ) ( ) ( ) ( ) ( ) ( )g g g g o g oε ε ε ε ε ε− = + + = +0 01

20

1

202 2 2 2' ' ' ' '

( ) ( )g gε − <0 0 ε ( )g' ' 0 0<

( ) ( )( )

( )( ) ( )g

d g

d

d f x d

dd f x dT' '

**0

0 00

2

2

2

22=

==

+

== ∇ ⋅ ≥ε

ε ε

ε

ε ε

Ω x

( )∇ ≥2f x* 0 d #

( ) ( ) ( )f x x x x1 2 12

223 1, = + + +

& ' ( ) [ ] [ ]x x1 2 0 1 0 1, , ,∈ ×

( (-3,-1)

#

( ) ( )( )∇ =

=++

f x x

f

xf

x

x

x1 21

1

1

2

2 3

2 1,

∂∂∂∂

( ) [ ] [ ]x x1 2 0 1 0 1, , ,∈ ×

#

( )( )

( )

( )

( )∇ =

=

>21 2

21 2

1 12

1 2

2 1

21 2

1 22

1 2

2 2

2 0

0 20f x x

f x x

x x

f x x

x x

f x x

x x

f x x

x x

,

,

,

,

,

∂∂ ∂

∂∂ ∂

∂∂ ∂

∂∂ ∂

x1 20 5 0 5= =. , . x

( )∇ <f d0 5 0 5 0. , . d " #

[7,3]

( )∇ <f d0 5 0 5 0. , . [ , ]− −1 1

%

*

x1 20 5 0= =. , x

( )∇ <f d0 5 0 0. , d

( ) [ ]∇ =

= +f dd

dd d0 5 0

7 27 21

21 2. ,

( ) [ ] [ ]x ε ε= + −0 5 0 1 1. , , −1 1

ε ≤ 0 5. ( ) ( ) ( )[ ] [ ]x x xε ε ε ε ε= = −1 2 0 5. d

( ) [ ]∇ =−

= − <f d0 5 07 2 1

15 0. ,

x1 20 0= =, x

! " ( )∇ ≥f x d* 0 #

x1 20 0= =, x

" # $

x* "f C∈ 2

# f "% #

x* d Ω ! f

( )∇ >f x d* 0 &

( )d f x dT∇ >2 0* ( )∇ =f x d* 0

Ω f x*

x* !

'

( )x x dε ε= +* ε0 d

( )( ) ( )f x gε ε= "0 0≤ ≤ε ε #

( ) ( ) ( ) ( ) ( ) ( ) ( )( ) ( )g g g o f x d o f x f xε ε ε ε ε ε− = + = ∇ ⋅ + = −0 0' * *

g d ( )∇ >f x d* 0

( )f x d ε = 0

x*

( )d f x dT∇ >2 0* ( )∇ =f x d* 0 d

( ) ( ) ( ) ( ) ( )( ) ( )g g g o f x f xε ε ε ε− = + = −01

20 2 2' ' *

! ε ( )d f x dT∇ >2 0*

x*

x*

!

! !

! (

!

"! #

)

! *

!

%+

x x1 2, ∈Ω Ω

Ω ( )θ θx x1 21+ − 0 1≤ ≤θ

Ω f(x)

! x x1 2, ∈Ω

( )( ) ( ) ( ) ( )f x x f x f xθ θ θ θ1 2 1 21 1+ − ≤ + −

"

#-f(x$ Ω f(x)

Ω ( )f x2 ( )f x1

Ω ( ) ( )a f x a f x1 1 2 2+ a a1 2,

!0 1≤ ≤θ x x1 2, ∈Ω

( )( ) ( )( )( ) ( ) ( )[ ] ( ) ( ) ( )[ ]

a f x x a f x x

a f x f x a f x f x

1 1 1 2 2 2 1 2

1 1 1 1 2 2 2 1 2 2

1 1

1 1

θ θ θ θ

θ θ θ θ

+ − + + − ≤

≤ + − + + −

! %

( )( ) ( )( )( ) ( )[ ] ( ) ( ) ( )[ ]

a f x x a f x x

a f x a f x a f x a f x

1 1 1 2 2 2 1 2

1 1 1 2 2 1 1 1 2 2 2 2

1 1

1

θ θ θ θ

θ θ

+ − + + − ≤

≤ + + − +

!

& Ω f(x)

( ) Γ = ≤x f x c

Γ x x1 2, ∈Γ

!0 1≤ ≤θ ( ) ( )f x c f x c2 1≤ ≤,

( )( ) ( ) ( ) ( )f x x f x f x cθ θ θ θ1 2 1 21 1+ − ≤ + − ≤

c '

Γ Γ

&

Ω ( )f x C∈ 1

!x y, ∈Ω

( ) ( ) ( )( )f y f x f x y x≥ + ∇ −

! 0 1< ≤α

( )( ) ( ) ( ) ( )f y x f y f xα α α α+ − ≤ + −1 1

! &

( )( ) ( ) ( ) ( )f y x f xf y f x

α αα

+ − −≤ −

1

! α → 0

( )( ) ( )( )( ) ( ) ( )Lim

f x y x f xf x y x f y f xT

α

αα→

+ − −= ∇ − ≤ −

0

'

( ) ( ) ( )( )f y f x f x y x≥ + ∇ −

' α x x1 2, ∈Ω

( )x x x= + −α α1 21 [0,1]

! ( ) ( ) ( )( )f y f x f x y x≥ + ∇ −

(

( ) ( ) ( )( )( ) ( ) ( )( )

f x f x f x x x

f x f x f x x x

1 1

2 2

≥ + ∇ −

≥ + ∇ −

! ( )1− α α

( ) ( ) ( ) ( ) ( )( ) ( ) ( )( )

( ) ( ) ( ) ( ) ( ) ( )( )( ) ( ) ( ) ( )

α α α∇ α

α α α∇ α α

α α

f x f x f x f x x x f x x x

f x f x f x f x x x x

f x f x f x

1 2 1 2

1 2 1 2

1 2

1 1

1 1

1

+ − ≥ + − + − ∇ −

⇒ + − ≥ + + − −

⇒ + − ≥

"

&

"

Ω ( )f x C∈ 2

F(x)

! y &

( ) ( ) ( )( ) ( ) ( )( ) f y f x f x y x y x x y x o x yT= + ∇ − + − − + −1

22F

f(y)

( ) ( )( )f x f x y x+ ∇ −x

ε x n y

( ) ( )( ) 1

22y x x y x o x yT− − >> −F

( ) ( )( )1

20y x x y xT− − >F

y ! o x y− 2

( ) ( ) ( )( )f y f x f x y x> + ∇ − ( ) ( )( )1

20y x x y xT− − >F

x !

y ( ) ( ) ( )( )f y f x f x y x> + ∇ −

" " !( ) ( )( )y x x y xT− − <F 0

( ) ( )( ) 1

202y x x y x o x yT− − + − >F x y

! # ! o x y− 2

#

!

Γ Ω⊆ ! Ω f(x)

!

c0 f

$ ! ( ) Γ = ≤y f y c0 "

%

x2 ! x1

( ) ( )f x f x2 1<

( )α ∈ 0 1, f

( )( ) ( ) ( ) ( )( ) ( ) ( )[ ] ( ) ( ) ( )

f x x x f x f x

f x f x f x f x f x f x

1 2 1 1 2

1 2 1 1 2 1

1+ − ≤ − + ≤

≤ + − < ≤

α α α

α max ,

α

x1 !x1

! "

x* ∈Ω Ω ( )f x C∈ 1

! ( )( )∇ − ≥f x y x* * 0 y ∈Ω

" !

( ) ( ) ( )( ) ( )f y f x f x y x f x≥ + ∇ − ≥* * * *

! y !

y !$& " !

( )∇ ≥f x d* 0 !d

!

'

Ω x

x x y y1 2, ∈Ω

"

!

! Ω f(x)

Ω !

x* x* ∈Ω

!x* # !y1

! ( )x y y* = + −α α1 21 x* y y1 2, ∈Ω !y2

( ) ( )( ) ( ) ( ) ( ) ( ) ( ) f x f y y f y f y f y f y* max ,= + − ≤ + − ≤α α α α1 2 1 2 1 21 1

x*

!" "

!Ω

" ( )f x y x y x y, = + + −

( ) Ω = − ≤ ≤x y x y, ,1 1

! ! Ω Ω #

! # & (&(&$

)

! !

!

! !

!

*

(

(

$

* ( ( $X

Y

!" #

$

#

#

%

&

# A

A

#

y x A

x A xk k+ =1

'(

K K

& $

x ∈Ω A

Luenberger

A x x= +1 A

# ( )f x x= −2 618 4.

& )*+, x0 01 1 10 100= . , , ,

0 2 4 6 8 1010

-1

100

101

102

'* xk '*

#

*(

#

( ) ( )f x x x

x x

x= =

− + >≤

; A0 5 1 1 1

0 5x 1

.

.&

'*

* * x∞ = 0

( *

0 2 4 6 8 10-10

-5

0

5

'* xk ')

A

Γ Ω⊆

-

&

A Γ Ω⊆

Ω Z

&

( ) ( )Z A x Z x< x ∉Γ *

( ) ( )Z A x Z x≤ x ∈Γ )

f(x)

( )∇ f x

')

A

Ω A

y A x= y A x yk k k≡ →

→∞ x xk k→∞

→

A

#

$

( ) x y x A x, ,∈ =Ω y x,y

A

# C=BA # B

& ')

( )

A xx x

x

A

A=

− + >≤

⇒+ = +

− = −0 5 1 1 1

0 5x 1

1 1 0 5

1 0 5 0 5

.

.

.

. .

ε εε ε

& ( ) x y x A x, ,∈ =Ω y

' '

[0,1] A xx x

x

x= − ≠

=

−2 0

1 0

1/&

&

x

y

0.5

1

1

x xk kk

kk=

∞ −∞

→∞= = =0 2 0 ; x lim

y A xk k k kk k

=∞

=∞ − −

∞= = − =0 022 2 0 ; y

A x A y∞ ∞= = ≠0 1

( ) x y x A x, ,∈ =Ω y A

Ω A ! "

Γ Ω⊆ x A xk k+ =1 # xk k=∞

1 x0

S ⊆ Ω xk k=∞

1 $

! " Z %

& A

xk k=∞

1

x

y

1

1

x xk k ∈ ℵ xk k=∞

1

x

x ( ) ( )Z x Z xk k k=∞

→∞ →

1

! " Z x xk k ∈ ℵ →

ε > 0 ( ) ( )Z x Z xk k ∈ ℵ → ( ) ( )Z x Z xk k ∈ ℵ →

Z ( ) ( )Z x Z xk0− < ε k0 ∈ℵ

( ) ( )Z x Z xk − ≥ 0 ! " k k≥ 0

( ) ( ) ( ) ( ) ( ) ( ) ( ) ( )Z x Z x Z x Z x Z x Z x Z x Z xk k k k k− = − + − ≤ − <0 0 0

ε

( ) ( )Z x Z xk k k=∞

→∞ →

1 ! ( ) ( )Z x Z xk k−

0 "

x x

" ( ) ( )Z A x Z x< Z A

" xk k+ ∈ ℵ1 !

x ( ) A xk k∈ ℵ !$

A !" $

x

x x

A x x

A xk k

k k

∈ ℵ

∈ ℵ

→

→

=

x ( ) ( )Z A x Z x<

'

A

$ $ ( )f x x=

!$ " A

( ) ( ) ( )f x x x= − ⋅ +1 12 2

#

( ) ( )xkk k= − ⋅ +−1 2 1

( )y xk kk

k= = + →−

→∞222 1 1

( )z xk kk

k= = − + → −−

− −→∞2 1

2 12 1 1

f(x) x

$

! " x ( )A x x= +1

∀ − > −x x x2 618 16184 4. . #

x x

r

(

#

)

-2 -1 0 1 20

1

2

3

4

5

) '

xk k=∞

1

x∞ x A xk k+ =1 #

e x xk k= − ∞ # k

!order of convergence" x∞ xk k=∞

1

p #

0 1 1≤−

−= < ∞

→∞+ ∞

∞ →∞+lim lim

k

k

kp k

k

kp

x x

x x

e

e

# p

! " p

! " p

p

lim !

lim

"

p ##

p

!

lim limk

k

kp k

k

kp

x x

x x

e

e→∞+ ∞

∞ →∞+−

−= =1 1 β

$

e fk kk

k kk

=∞ −

=∞ −= =1 12 3 ;

!

P

β

%

lim lim lim.

k

k

kp k

k

kp k

k

pk

x x

x x

e

e

p

p

p→∞+ ∞

∞ →∞+

→∞

− −

−−

−= = =

==

∞ =∞ =

1 112

2

0 0

0 5 1

2

0 3

& " p=1

p=1 '( p

! $ ε p p= + ε

lim lim . limk

k

k k

k

k k k

ke

e→∞++ →∞

− −

− − →∞=

⋅= ⋅ = ∞1

1

12

2 20 5 2ε ε

ε

0.5 β p=1

0.3333 β

! p=2 ( )ek k

k

=∞ −

=1

22

( )( )

( )( )lim lim lim lim

k

k

kp k

k

kp k p k

px x

x x

e

e

p

p

p

k

k

k

→∞+ ∞

∞ →∞+

→∞

−

− →∞

− −−

−= = = =

===

∞ =

+

1 12

2

2 22

2

2

0 0

0 1

1 2

0 3

1

'

$ '

! x∞ xk k=∞

1

lim limk

k

k k

k

k

x x

x x

e

e→∞+ ∞

∞ →∞+−

−= = <1 1 1β

β

e ckk= β "

$

)

* '

! p=1 ( )e kk kk

=∞ =1 1 /

lim lim limk

k

kp k

k

pk k

p ke

ek

k

k

k k

p

p

p→∞+

→∞

+

→∞= +

=+

⋅

+

=

==

∞ =∞ =

1

11

1

1 1

1

1

0 0

0 1

2

0 3

limk

kk

k e→∞ +

=1

1

ekk = 1

2

$ ' & '

$

k

" &

( ) ( )( ) ( )lim lim

?

k

k

k k

k

k

x x

x x

f x f x

f x f x→∞+ ∞

∞ →∞

+ ∞

∞

−−

= < ⇒−

−=1 1

1β β

+

x∞ ( ) ( )f x x x x, , ,∞ ∞ g

! b a x

(*

( ) ( ) ( )0 < ⋅ ≤ ≤ ⋅ < ∞∞ ∞ ∞a g x x f x x b g x x, , ,

β

!

( )f x x x x, ∞ ∞= − 2'

( ) ( ) ( )g x x x x Q x xT, ∞ ∞ ∞= − − ,

$ $ Q

Q $ b a

$

!

( ) ( )λ λmin maxx x x x Q x x x xT− ≤ − − ≤ −∞ ∞ ∞ ∞2 2

$$ $ $

$$ $

$ $ $

$ $ $

$

x

! " d

"

!

! "

dk

α k

f(x) f(x)

Xk X X dk k k k+ = +1 α

#

d

$ ! "

%

S E En n: 2 →

( ) S x d y x d f x d, : , arg min= = + = +

y α α αα

0 0

d ≠ 0 Sx,d En f

d ≠ 0 (x,d) dk k=∞

1 xk k=∞

1

( )f x dk k+ α α0k ( )x dk k,

y S x y yk k k k= →

→∞, y x dk k

kk= + α0

y=Sx,d

%

α α0 11

0k

kk k

k kk

y x

d

y x

d=

∞

=

∞

→∞∞= −

→ = −

d ≠ 0 "

%α y x d= + ∞α0 !

( ) ( ) ( ) ( )f y f x d y f x dk k k≤ + ⇒ ≤ +α α f

S y S x d= , ( ) ( )f y f x d≤ +minα

α

! "

&

d ≠ 0

x (x,0) Sx,y ( ) ( )f x x= −1 2

% dkk

k= →−

=

∞2 0

1 !x=0 "

( )

S d y x d f x d

y y

k

kk

0

0 2 2 1

0 0

0 0

, : , arg min

: ,

= = + = +

=

= = + = = =

y

y -k

α α α

α α

α

% y =1

( ) S y x d f x d y0 0 0 0 0 00 0 0 0, : , argmin : ,= = + = +

= = + = = y y α α α α αα

d=0

S E En n: 2 → En f ! "

( )∇ ⋅ =f S x d d, 0

d

( ) ( )∂ α

∂ααf x d

f x d d+ = = ∇ + ⋅0 0 α0

x0 x f %

f(x) xk k=∞

1 A

( ) ( ) ( ) ( )f x f x f x f xn1 2 3> > > ≥ ≥

'

#

$ A

A

! "

!uni-modal" 0,C

Xk

Xk+1

dk

f(x)

0,C

xk kN

=1 N

!" #

$ % & '

' ! ( &

$ %

#

)

) * +

,

,

' ,

F F F F

F

n n n0 1 1 2

0 1 2 3

1= = = +

⇒ =

− − F

F F F F 1, 1, 2, 3, 5, 8, 13, 21, 34, 4, , , , ,

!

&

!% ,

,

- •

* + • ,

\ \

.

,

, $

# &!( •

C C B A

AC

CB

CB

AB=

b AC a BC

'

b

a

a

a bab a=

+⇒ + − = ⇒

b a =1

b = 0.618 or -1.6182 2 0

A C B

/

'

, &!(

τ τ τ2 1 0− − = ' •

, .

τ τ

τ τ

τ τ

τ τ

2

3

4

1 2

1

2 1

3 2

= +

= +

= +

= +− −

nn kF F

,

http:\\www.mcs.surrey.ac.uk\personal\R.Knott\Fibonacci

&

'

εkN k

N

F

FC=

⋅− +1

N=2,3,5

F

F

F

F1

2

1

21 0 5

= −

= . N=2

0.5 x x2 10 5 0 5= − = +. , .δ δ

*a<b + b a

max ,b a1−

0.5 b 1-a b +

*

(

1

ε1 = C

F

F CNN

−

1 2

1 1−

−FF CN

N

3

ε21=

−FF CN

N

4

N 5

6

εN N

FF C=

1

, ' &

+ F

F

F

F2

3

2

3

2

31

1

3

= −

= ; N=3

0.666666 * 0.3333

0.3333

F

F

F

F4

5

4

5

5

81

3

8

= −

= ; N=5

2/8 3/8 5/8

1/8 3/8

N

0.618

F F Fn n n= +− −1 2

!"

F A Bnn n= +τ τ1 2

"

τ τ τ τ21 21 0 0 618 1 618− − = ⇒ = − , . , . τ τ1 2,

1.618

! N

lim.

.N

N

N

F

F→∞−

= = =1

11

16180 618 τ

ε τkk C≅ ⋅−

11

0.618

N

#$%&

1 2

1 3 2

4 5 3 2 1

N=2

N=3

N=5

$#

' (

) *

(

f

( ) ( ) ( )f x f x f xk k k, ' , ' ' xk

!

( ) ( ) ( )( ) ( )( )q x f x f x x x f x x xk k k k k= + − + −' ' '1

22

! q

( ) ( ) ( )( ) ( )( )q x f x f x x x x

f x

f xk k k k k k kk

k' ' ' '

'

' '+ + += = + − ⇒ = −1 1 10 x

xk ) &

( )q x' = 0 f(x)

g(x)=0 g(x)

xk+1 xk xk

!" * '

( ) ( )( ) ( ) ( )( )L x f x x x f x x

f x

f xk k k k kk

k= − + ⇒ = −+'

' x 1

$%

x0

( ) ( )f x f x' , ' '0 0

xk k

( ) ( )f x f xk k' , ' '

( )( )x x

f x

f xk kk

k+ = −1

'

' '

&

"

)

+

,

xk

xk+1

x * f

( ) ( )f x f x' * , ' ' *= >0 0

xk k=∞

1 x* x0

( )

22− k

( x*

( ) ( )( )

( ) ( ) ( )( )( )x x x x

f x f x

f x

f x f x f x x x

f xk kk

k

k k k

k+ − = − −

−= −

− − −1 * *

' ' *

' '

' ' * ' ' *

' '

! " θ ∈ [ , ]0 1 xk ( )f x' *

( ) ( ) ( )( ) ( )( )( )

( ) ( ) ( )( ) ( )( )( )

f x f x f x x x f x x x x

f x f x f x x x f x x x x

k k k k k

k k k k k

' * ' ' ' * ' ' ' * *

' ' * ' ' * ' ' ' * *

= + − + + − −

⇒ − − − = − + − −

1

21

1

21

2

2

θ θ

θ θ

xkxk+1 xk xk+1

x xk k= +2xk+1

#

! $

( ) ( ) ( )( )( )

( )( )( ) ( )x x

f x f x f x x x

f x

f x x

f xx xk

k k k

k

k

kk+ − = −

− − −=

+ −−1

21

2*

' ' * ' ' *

' '

' ' ' *

' '*

θ θ

!x* xk

( )( )( )

∃ < < ∞ + − <

∃ < < ∞ > >

0 1

0 0

1 1

2 2

k f x x k

k f x k

k

k

' ' ' *

' '

θ θ

!

( )( )

( )( )( )

x x

x x

f x x

f x

k

kk

k

k

k

+ −

−=

+ −<1

21

2

1

2 2

*

*

' ' ' *

' '

θ θ

%

&

k "

! x xk kmin max, &xk+1

[ ] [ ] [ ]x x

x x x x x

x x x x x

k kk

kk

k k

k kk k k

min maxmin max

min max

,max , ,

,min ,=

>

<

+

+

1

1

' $

"

( ) ( )f x f xk k' , ' −1 xk

!

(

( ) ( ) ( )( ) ( ) ( )( )q x f x f x x xf x f x

x xx xk k k

k k

k kk= + − + ⋅

−−

−−

−'

' '1

21

1

2

( ) ( )f x f x

x xk k

k k

' '−−

−

−

1

1

! q(x)

( ) ( )( ) ( )x x

x x f x

f x f xk kk k k

k k+

−

−= −

− ⋅−1

1

1

'

' '

"

!

x x0 1,

( ) ( )f x f x' , '0 1

xk k

( ) ( )f x f xk k' , ' −1

( ) ( )

( ) ( )x xx x f x

f x f xk kk k k

k k+

−

−= −

− ⋅−1

1

1

'

' '

)*

+ 1.618

&,

"

"

"

&+ ))-

!

ε ε ε εk k k k kx x= − ⇒ = ⋅− −* 1 2

( ) ( ) ( )L k L k L kε ε ε= − + −1 2 Log

( )L kε

exp .0 618k εk 0 618. k

x x xk k k , ,− −1 2 !

"

( ) ( ) ( )( )( )( )

( ) ( )( )( )( )

( ) ( )( )( )( )

q x f xx x x x

x x x x

f xx x x x

x x x x

f xx x x x

x x x x

kk k

k k k k

kk k

k k k k

kk k

k k k k

=− −− −

+

+− −− −

+

+− −− −

− −

− −

−−

− − −

−−

− − −

1 2

1 2

12

1 1 2

21

2 2 1

! # ! $

1.3

!

! % %

! ! %

# $

&

x x x0 1 2< <

( ) ( ) ( )f x f x f x0 1 2≥ ≤

k

x x x0 1 2< <

( ) ( ) ( )f x f x f x0 1 2≥ ≤

q(x)

x*

( )f x * !

x x x' ' '0 1 2< < x* x x x0 1 2< <

x*

'

(

! ) %

% !

*

α ! ! )

c <<1 ) % α∞ %

! % α α α− ≤∞ ∞ c

! !

"k !

α α αα α

αα α

α∞∞

∞∈ ⇒

−≤ − ≤k k

k k

kcmin max

max min

min,

% ! %

!

! )

# $ !

"

!

# $

!!

!

%

! ! ! &

! ! !

!

!

! !

!

0.61821.6181.31

! ! +

x x xn1 2, , , n

x x xn10

20 0, , ,

x1 !

!

ei

Aitken Double Sweep Method! n "

" n-1 n "

#

Gauss-Southwell Method

Simplex

!

Simplex

#

!

Simplex

$%

"

x x x xn0 0

102

0= , , ,

k

k=n k=1

γopt

( )f x x x x xk k kk

kk

kn

− − −−

−+

−11

12

11

11

1, , , , , , , γ

x x x x x xk k k kk

opt kk

kn= − − −

−−+

−11

12

11

11

1, , , , , , , γ

k=2n-1 k=n+1

!"

#

" " 2n-1

" !"

Aitken Double Sweep Method &

$"

Simplex xk kn

=0 n+1 Simplex En

( )x n xcenter kk

n= + ∑−

=1 1

0 Simplex

! Simplex

'

Simplex Simplex

( Simplex

f(x) xk kn

=0 En Simplex

)

Simplex $

x x f x fmin min max max, , , ,

x x xref = −2 max !

xmax ! xnew #

( ) ( ) ( ) ( )

[ ] ( ) ( )[ ] ( ) ( )

x

x x f x x f x

x otherwisef x f x

x otherwise

x x f x f x

x x otherwisef x x f x

new

ref ref ref

refref

ref

ref

ref jj j ref

=

− − <

>

+ ≤+

≠ ≤

2 2

2

2

/

/max

min

max maxmax x

Simplex %

1− ≤f fmin max/ ε &

Simplex (

$&

( ) ( ) ( ) ( )

x f x f x

x f x f x

xn

x x

xj

xj

xj

xj

jj

n

j j

j j

min min

max max

max

argmin min

arg max max

= =

= =

= −∑

=

; f

; f

1

0

Simplex *

simplex!

D C B A Simplex *

Simplex !

xmax

x xref new=

x x xrefexp = −2

xmax

x xref new=

xmax

xref

xmax

xref

xnew

xnew

C D

A B

$(

"

f(x) & f(x)

(

\

)

( ) ( ) ( )[ ] ( ) ( ) ( )[ ]∂∂ δ

δ ∂∂ δ

δ δf x

xf x e f x

f x

xf x e f x e

jj

jj j≈ + − ≈ + − −1 1

2

e j x j x j

n+1 ! 2n

)

( ) ( )[ ] ( )

( ) ( )[ ] ( )

1 1

1

2

1

22

δδ

∂∂ δ

δ

δδ δ ∂

∂ δδ

f x e f xf x

xe o

f x e f x ef x

xe o

j j

j j j

+ − = +

+ − − = +

)

( ) ( ) ( ) ( )

( ) ( ) ( ) ( )

f x e f xf x

xe e

f x

xe o

f x e f xf x

xe e

f x

xe o

j j jT

j

j j jT

j

+ = + + +

− = − + +

δ ∂∂

δ δ ∂∂

δ

δ ∂∂

δ δ ∂∂

δ

1

2

1

2

22

22

22

22

Limo

α

αα→

=0

0

#

)

$*

( ) ( ) ( ) ( )[ ]∂∂ δ

δ δ2

2 21

2f x

xf x e f x e f x

jj j≈ + + − −

x

) !

( ) ( ) ( ) ( ) ( ) ( ) ( )[ ]( ) ( ) ( ) ( ) ( )[

( ) ( )]

∂∂ ∂ δ

∂ δ

∂∂∂ δ

δ δ δ δ

∂∂ ∂ δ

∂ δ

∂

∂ δ

∂ δδ δ δ δ

δ δ δ δ

2

2

2

2

1 1

1

2

1

4

f x

x x

f x e

x

f x

xf x e e f x e f x e f x

f x

x x

f x e

x

f x e

xf x e e f x e e

f x e e f x e e

i j

j

i ii j j i

i j

j

i

j

ij i j i

j i j i

≈+

−

= + + − + − + +

≈+

−−

= + + − + − −

− − + + − −

δ

#

Scale δ

δ

δ )

#

d

( )( )

d f x

x x f x

k kT

x k kT

k k k k x

k= − ∇ > =

= − ∇+

M M M M

M

0

1

,

α

M ! M

M M= ∑ ⇒ = ∑= =

λ λk kk

nkT

k k kk

nu u v u v u

1 1 ,

M

v u v uk kk

n= ∑

=,

1

uk

M

"

α k >0 ( )d f xk kT

k k= − ∇ >M M 0

ε2 M=I f

( ) ( ) ( )( ) f x f x f x x x o x xk k k k= + ∇ − + −

( )x x f xk k k k x+ = − ∇1 α M

( ) ( ) ( ) ( ) ( ) f x f x f x f x o f xk k k kT

k kT

k+ = − ∇ ∇ + ∇1 α αM M

( ) ( )∇ ∇ >f x f xkT

kM 0 . α k → 0

( ) ( )

( )

α ε

λ

ε α λ

k kT T

k

Tk j j

j

j

jT

k j jj

n

f x M M f x

v f x u u

u v

2 2

1

2 2 2 2

1

∇ ∇ =

= ∇ = ∑

⇒ = ⋅ ∑

=

=

,

,

M

!

( ) ( ) ( )∆k k k kT

k k j jj

nM x f x M f x u v, ,= ∇ ∇ = ⋅ ∑

=α α λ

2

1

# ,λ α λk j k j=

Max u v t u vk j

k j jj

nk j j

j

n

, ,,

, . . ,λ

λ ε λ j=1...n

s 2

1

2 2 2

1= =∑

= ∑

$

Max u v u vk j

k j j k j jj

n

j

n

, ,,

, ,λ

λ µ ε λ j=1...n

2 2 2 2

11+ − ∑

∑

==

u v n j k j k j, ,2, ,...,

, ,2

1 2 0 1 31

2− = = ⇒ =µλ λ

µ for j

#

ε λµ µ

µε

λ ε2 2 2

12

2

12

1

4

1

4

1

2= ∑ = =∑ ⇒ = ⇒ =

= =

, ,

, ,k j jj

nj

j

n

k ju v u v

M

( )d f xkT

k= −∇

α αk const= = Mk I=

α k Steepest Descent

Normalized SD

α k = 1 % & ( )M Fk kx= −1

Mk

%Diagonal NSD& DNSD k

'

( )f x x x x b cT T= − +1

2Q

Q

! !

Qx*=b

%Q &

( ) ( ) ( )f x x x x b c x x x x x x cT T T T= − + = − − − +1

2

1

2

1

2Q Q Q* * * *

% () \ * Q &

N EEG Z

Z j j

L

=1 L

%& Cj Z C Z nj j j= +

! Z n j

( ) ( ) ( )f Z Z C Z Z C Zj jT

j jj

L= − −∑

=1

!

+

( ) ( ) ( )f Z Z C Z Z C Z Z Z Z b c

C C C Z Z Z

j jT

j jj

L T T

jT

jj

LjT

jj

LjT

jj

L

= − −∑ = − +

= ∑ = ∑ = ∑

=

= = =

1

1 1 1

1

2

2 2

Q

Q ; b ; c

!

LS %Least Squares&

!

! f(x)

( ) ( ) ( )( ) ( ) ( )( )( ) [ ]

f x f x f x x x x x x x

where x x

T= + ∇ − + − −

= + − ∈

0 0 0 0 0

0

1

21 0 1

F ξ

ξ θ θ θ: ,

Q b

%,, & %Least-Squares&

SD

( )∇ = −f x x bQ x

x x b xk k k+ = + −1 α Q !

SD SD

( )α λ∈ 0,2 / max Q

maxk

k1− αλ Q

SD

x I x b I x I bk kk j

j

k+

+

== − + = = − + ⋅ −∑1

10

0α α α α αQ Q Q

x b* = −Q 1

[ ] [ ]

[ ][ ]

[ ] [ ][ ]

[ ] [ ]

x I x I x

I x I I Q Qx

x I x x

kk j

j

k

A I A I A

k k

k

j k

j

k

++

=

↑= − −∑

+ + −

+

= − + ⋅ −∑ =

= − + − − =

= + − −

+ −

=

11

00

10

1 1

10

1 1

1

α α α

α α

α

Q Q Q

Q Q

Q

*

*

* *

[ ]det 1 0− ≠A !

! " I k− +αQ 1 !

" ! k ! k

! ∆ Q U U= * ∆ ! ! Q

! U Q !

U I U I* − = −α∆ αQ

1− = −αλ λ αk k IQ Q I − αQ !

− < − < ⇒ < <1 1 1 0

2αλ αλk Q

Q

max

" α

#

[ ] [ ][ ]x U I Ux U I I Uxkk k

++ += − + − −1

10

1* * *α∆ α∆

x* " !

! !

( ) [ ] [ ]( ) [ ] [ ]

( ) [ ] ( )

U x x I Ux I Ux

U x x I Ux I Ux

x x I U x x

kk k

kk k

k k

++ +

+

− = − − −

− = − − −

⇒ − = − −

11

01

0

1

* *

* *

* *

α∆ α∆

α∆ α∆

α∆ U

x x I x xx x

x xIk k

k

k kk+

+− ≤ − − ⇒−

−≤ − = −1

1 1* **

*maxα∆ α∆ αλ Q

"

"

[ ]e I ek k+ ≤ −1 α∆ [ ]e U x xk k= − * !

[ ]1− αλ j Q [ ]I − α∆ !

" ek !

# #

! ! !

! [ ]1− αλ j Q !

! α $

α ! %# ! 1− αλ k

! " "

α αλα

optk

k= −min max 1 Q

&

SD α ! %# !

"! α ! max... !

1 12− = − ⇒ =+

α λ α λ αλ λopt opt optmin max

max min

12 1

1−

+= −

+= −

+↑=

λλ λ

λ λλ λ

λλ

min

min max

max min

min maxmax

minr

r

r

SD α SD

Q !

! !

'

( )λ max max ,≤ ≡ ∑∞Q Q i ji j

%& ! SD

α

1− αλ k

#

1λ min

1λ max

1λk

α

(

x0

α

k

x x b xk k k+ = + −1 α Q

( )b x f xk k− = ∇ ≤Q ε

ε

SD %& !

! !

%( !

α "

!

x0

!

α

k

( )x x f xk k k+ = −1 α∇

ε ( )∇ ≤f xk ε

SD %( !

)

NSD

! ! α k "! NSD

! ! "

α k

α k x x ek k k k+ = +1 α NSD

α kkT

k

kT

kk k

e e

e eb x= = −

QQ where e

! " "! α k

( ) ( ) ( ) ( )f x e x e x e b x ek k k k k kT

k k kT

k k k+ = + + − +α α α α1

2Q

α k

( ) ( )∂∂α

α α

α

kk k k k

Tk k k k

T

kk

Tk

kT

k

f x e e x e e b

e e

e e

+ = + − =

⇒ =

Q

Q

0

! !

! ' ! e ekT

k+ =1 0 "!

"!

[ ] ( )[ ] [ ]e e e b x e b x e e e ekT

k kT

k kT

k k k kT

k k k+ += − = − + = − =1 1 0Q Q Qα α

!

" ! NSD

NSD

NSD

Condition # r r

r

−+

1

1 ! "

Q

( ) ( ) ( )E x x x x xT= − −1

2* *Q

#

( ) ( )( )

( ) ( ) ( ) ( )( ) ( )

( ) ( ) ( ) ( )( ) ( )

E x E x

E x

x x x x x e x x e x

x x x x

e e e x x

x x x x

k k

k

kT

k k k kT

k k k

kT

k

k kT

k k k kT

k

kT

k

−=

− − − + − + −

− −=

= −+ −

− −

+1

2

* * * *

* *

*

* *

Q Q

Q

Q Q

Q

α α

α α α

# α k e b x x xk k k= − = −Q Q *

( ) ( )( )

( ) ( )[ ]

[ ][ ]E x E x

E x

e e

e e

e e

e e

e e

e e

e e e e

k k

k

kT

k

kT

k

kT

k

kT

k

kT

k

kT

k

kT

k kT

k

−= −

−=+

− −1

2 2

1

2

1

2Q Q

Q Q Q

$

# Kantrovitch

( ) ( )( )

[ ][ ][ ] ( )

E x E x

E x

e e

e e e e

aA

a Ak k

k

kT

k

kT

k kT

k

−= ≥

++

−1

2

1 24

Q Q

# Q a A

( )( )

( ) ( ) ( )E xaA

A aE x

A a

A aE x

r

rE xk k k k+ ≤ −

+

= −+

= −+

1 2

2 21

4 1

1

# % a A r

( )( )

E x

E x

r

r

x x

x x

r

rk

k

k

k

+ +≤ −+

⇒−

−≤ −

+

12

11

1

1

1

*

*

r Q

α SD

&

SD NSD

SD &

& Qx b* = & !NSD "

! "

' ( ( )g x x b= −Q 2

Q & g

# Q2

( ) ( ) ( ) ( )( ) ( ) ( ) ( )

f x x x x xr

rr

g x x b x br

rr

T

T

= − − ⇒ −+

≈ −

= − − ⇒ −+

≈ −

−

−

0 51

11

1

11

21 4

2

2

22 4

. * *Q

Q Q

r=100 &

r 0.9999 0.99

NSD

x0

k

α kkT

k

kT

kk k

e e

e eb x= = −

QQ where e

x x ek k k k+ = +1 α

( )e f xk k= ∇ ≤ ε

ε

NSD

$

NSD SD

#

x0

k

( )∇ f xk

α k

( )( )f x f xk k− ⋅ ∇α

( )x x f xk k k k+ = − ∇1 α

( )e f xk k= ∇ ≤ ε

ε

NSD $

( ) ( ) ( ) ( ) ( )( )

( ) ( ) ( )[ ]

f x x x x xf x x x

x

x x x f x x x x x

T= − − ⇒∇ = −

=

= − ∇ = − − =− −

1

2

1 01

01

0

* **

* *

QQ

F Q

F Q Q

SD &

x* f ( )f x C∈ 3

)

! "

( ) ( )( ) ( ) ( ) ( )( )[ ]

x x x x x f x

x f x f x x x x

k k k k

k k k k

+−

−

− = − − ∇ =

= − ∇ − ∇ + −

11

1

* *

* *

F

F F

#

*

( ) ( ) ( )( ) ∇ − ∇ + − = −f x f x x x x O x xk k k k* * *F 2

#

( )x x x k x x

x x

x xk k

k k k

k

k

+−

+

− ≤ ⋅ ⋅ −

⇒−

−≤

11

12

12 1 2

* *

*

*

F

& k2

"

!

x0

k

( ) ( )F x f xk k,∇

( ) ( )x x x f xk k k k+−= − ∇1

1F

ε ( )∇ ≤f xk ε

&

&

#

1

2

3

NSD

! M

α k = 1

δ x0

k

( ) ( )F x f xk k,∇

( )ε δk kI x I+ >F εk > 0

( )[ ] ( )d I x f xk k k k= + ∇−ε F1

α k !

( ) ( )g f x dk k kα α= −

!

( )[ ] ( )x x I x f xk k k k k k+−= − + ∇1

1α ε F

ε ( )∇ ≤f xk ε "

#

"#

Levenberg-Marquardt

Cholesky

$

%!

$

δ x0

k

( ) ( )F x f xk k,∇

( )εk kTI x GG+ =F Cholesky

$ G

εk > 0 %

( )ε δk kI x I+ >F

$ ( )GG d f xTk k= ∇

% G

α k !

( ) ( )g f x dk k kα α= −

"

( )[ ] ( )x x I x f xk k k k k k+−= − + ∇1

1α ε F

ε ( )∇ ≤f xk ε &

#

"

εk % Choleski % & '

DNSD

%NSD DNSD

n %

NSD %

(

( )x x f x x e x ek k k k k k k k

k

kn

k+ = − ∇ = + = +

1

1 0

0D D

α

α

! %NSD

Dk %DNSD

ek I ek k− =QD 0

! %

( ! α kj

j

n

=1

( ) [ ] [ ]f x x x x xk kT

k+ + += − −1 1 1* *Q

( α α αk k kn* = 1

%

( ) [ ] [ ]∂∂αf x

diag e x x ek

kT

k k k* *= − + =Q D 0

! f %

! f

%' !

( Q Qx x x b ek k k− = − = −*

&

[ ]diag e I ek k k− =QD 0

diag ek ek

% I ek k− =QD 0

(

1 0

1

0 1

0

0

1

1

11 12 1

21 22 2

1 2

1

2

1

1

2

111

112 1

121

122 2

−

=

=

− − −− − −

−

q q q

q q q

q q q

e

e

e

q q q

q q q

n

n

n n nn

k

k

k

k

k

kn

k k kn

n

k k kn

n

αα

α

α α αα α α

α

k n k n kn

nn

k

k

knq q q

e

e

e11

12

1

20

− −

=

α α

Q ! %

(

I e e e e ek k k k k k k k− = ⇒ = ⇒ =− −QD Q D Q0 1 1 α* ./

MATLAB )

! %Q !

Qx*=b

α k*

D Qk k ke e= −1

!

% % !

*

DNSD

( Q

( )( )

( )( )

Dk

k

k

k

k

nn

f x

x

f x

x

q

q

=

=

−

−

−

−

∂

∂

∂

∂

2

1 2

1

2

1 2

1

111

1

0

0

0

0

DNSD Q

Q % %

%

% !

SD

SD

NSD SD Condition # Q

!"

SD

The Conjugate

Gradients Method

CG

#!

NSD PARTAN

NSD

#$

NSD #$

SD DNSD

NSD PARTAN LM Newton

SD

%

(Parallel Tangent) PARTAN

!%& !!

x0 NSD

y1 ' NSD x1 NSD

x2 NSD x y2 1=

y1 x0

xk+1 yk NSD xk k

#( yk xk−1

NSD PARTAN #(

)

[ ]( )

[ ]

1 3

2 4

1 1

1 1

. . argmin

. .

e

y xk+1

k k k k k k

k kk

Tk

kT

kk k k k k

b x f y x x

xe e

e ee y x x

= − = − +

= + = − +

− −

− −

Q

Q

θ θ

θ

θ

) θk

θ θ θ

θ

θ

θk k k k

Tk k k

k kT

k k k

kk k

Tk

k kT

k k

k kT

k

k kT

k k

y x x x y x x x

y x y x x x

y x x x

y x y x

y x e

y x y x

= − + − − + −

⇒ − − + − =

⇒ =− −

− −=

−

− −

− − − −

− − −

− −

− −

− −

− −

arg min * *

*

*

1 1 1 1

1 1 1

1 1

1 1

1 1

1 1

0

Q

Q

Q

Q Q

x0

x1

y1 x2

y2

x3

y3

x4

SD

*

θk =1 x x y x y xk k k k* *− = − ⇒ =− −1 1

#& x y xk k+ = =1 *

NSD

NSD

' NSD

NSD

x0

NSD x1

NSD xk k

yk

θk x x yk k k−1 , ,

x y x xk k k k k+ − −= − +1 1 1θ

( )e f xk k= ∇ ≤ ε

ε

PARTAN #&

)

( )[ ] ( )[ ] ( )

x x b x x

I x x b

k k k k k k k

k k k k k k k

+ −

−

= + − + − =

= − + − +1 1

1

1

1

θ α θ

θ α θ α θ

Q

Q

SD

#

SD

SD

SD

) SD

x x e b x

x x x x x x I x x

k k k k k

k k k k

+

+

= + = −

⇒ − = − + − = − −1

1

α

α α

where e Q

Q Q Q* * * *

~ ~ ~ε α ε α εk kkI I+

+= − = −11

0Q Q ~ *εk kx x= −

)k

( ) ( )P z zk k jj

j

kk k j

j

k= ⋅∑ = =∑

= =α α, ,

0 01 1 ; P

NSD

)

y xk k j jj

k= ∑

=α ,

0

+ yk

PARTAN

)

( )

[ ] [ ]( )

y x x x x x

I P I

k k k j jj

kk j j

j

kk j j

j

k

k k j jj

kk j

j

j

kk

− = = ∑ − = −∑ = ∑

⇒ = ∑ = −∑ = −

= = =

= =

* * * ~

~

, , ,

, ,

∆ε α α α ε

ε α ε α α ε α ε

0 0 0

0 00 0Q Q

!

SD ' +

SD

SD

I − αQ

I − αQ

SD

( )( )( ) ( ) ( ) ( ) ( )

P z

P z z

P z z P z P zk k k k k k k

0

1 1 1

1 2

1

1

1 1 2

== − +

= − + + − ≥− −

γ γ

ρ γ γ ρ k

! "

# $ z=1

SD

[ ]

( )[ ] ( )

y I y b

y I y b yk k k k k k

1 1 0 1

1 1 2

= − +

= − + + − ≥−

γ α γ α

ρ αγ αγ ρ

Q

Q

kk-2

PARTAN

( )P z1 k=1

( )( )

( )

[ ]P z z

x x

x y

x x b x

y b

1 1 1

1 1 0 1 1

0 0

1 0 0

1 0 1

1

1 1

= − +

⇒ = − + =↑

=

= + −

− +

γ γ

γ γ

α

γ α γ

y

,

Q

Q

k

SD

"%%

[ ]( )[ ]( ) [ ]( ) ( ) [ ]( )

[ ]( ) ( )

ε α ε

ρ γ α γ α ρ α ε

ρ γ α γ ε ρ ε

k k

k k k k k k

k k k k k k

P I

I P I P I

I

= − =

= − − + − + − − =

= − − + + −

− −

− −

Q

Q Q Q

Q

0

1 2 0

1 2

1 1

1 1

x* #

( ) ( ) ( )( ) ( )

[ ]( ) ( )

y x x

y y x

b I y y

k k k k k k k k

k k k k k k k

k k k k k k

= − + + − + − + =

= − + + − + =

= + − + −

− −

− −

− −

ρ α γ ε ρ ε ρ ρ

ρ α γ ρ ρ γ α

ρ γ α α γ ρ

Q

Q Q

Q

1 1 1

1 1

1

1 2

1 2

1 2

* *

*

Qx b* =

ρk =1

αγkk

Tk

kT

kk k

e e

e eb y= = −− −

− −

1 1

1 1QQ where e

ρk αγ k NSD

PARTAN

yk

& x*

α SD

α=1 α γ⋅ k

NSD SD

Q ∀ ≠k k kρ γ, 0

x*

SD

"%"

( ) ( )ε ρ γ ε ρ εk k k k k kI= − + −− −Q 1 21

( ) ( )( )[ ]

ε ρ γ ε ρ ε

ρ γ ρ ε ρ γ ε∞ ∞ ∞

∞ ∞

= − + −

⇒ − − = ⇒ =k k k

k k k k k

I

I

Q

Q Q

1

0 0

Q ∀ ≠k k kρ γ, 0

x* ε∞=0

ρ γk k k, =∞

1

# $yk−1 ! k-1

ρ γk k,

[ ] [ ]

( )[ ] ( )

γ ρ

ρ γ γ ρ

γ ρk k k

Tk

k k

y x y x

where y I y b y

, * *,

= − −

= − + + −−

argmin

k-2

Q

Q 1 1

ρ γk k,

( )[ ]

( )[ ] ( )[ ]

( )[ ]

∂∂γ

ρ ργ

∂∂ρ

γ ρ ργ

γ

fe y y y e e b

fy y e y y y e x

y y e b

kT

k k k k kT

k k kT

k k k k

k k kT

= ⇒ − + + =

= ⇒ − + − + + − =

= − +

− − − − − −

− − − − − − −

− − −

0

0

1 1 2 2 1 1

1 2 1 1 2 2 1

1 2 1

Q

Q *

ρ γk k,

γk PARTAN PARTAN

NSD

NSD

SD

"%'

SD #

PARTAN NSD

#(

! PARTAN

$

SD

SD

( ) [ ] [ ]x x f x x x x e x xk k k k k k k k k k k k k+ − −= − ∇ + − = + + −1 1 1α β α β

# βk k kx x− −1

! NSD SD βk = 0 1

$α k

)" 2

[ ] ( )( )[ ]

y y b y y

y e y y

k k k k k k k

k k k k k k k

= + − + − =

= + + − −− −

− − −

ρ ρ γ ρ

ρ γ ρ1 1

1 1 2

1

1

Q k-2

α ρ γ β ρk k k k k= = − , 1

#

Condition # *

$ !

SD

)+

SD

NSD

MomentacceleratedAlgorithm

SD

! "

!

# Q µ j j

n

=1

$ # ! α β,

( )0

2 11< <

+− < <α

ββ

A 1;

Q A

$

x x e x x

x x x x e x x x x

x x

k k k k k

k k k k k k

k k k k

+ −

+ + −

−

= + + −

⇒ − = = − + + − − + =

= + − + −

1 1

1 1 1

1

α β

ε α β

ε α β ε ε

* * * *

* Q

SD

%

⇒ = + − −+ −ε β α ε βεk k kI1 1Q

$& ! "

( )v

I I I

Ivk

k

k

k

kk=

=+ − −

=−

−

−−

εε

β α β εε1

1

210

QT

" T !

$' (

( )det T

Q=

+ − −

−

=I I I

I

I

I

β α βλ

0

0

00

$ Q = U U* ∆ Q

( ) ( )

( )

det det

det * det det

det

T Q= − + − + + =

= ⋅ − + − + + =

= − + − + + =

λ β α λ β

λ β α∆ λ β

λ β α∆ λ β

I I I I

U U I I I I

I I I I

2

2

2 0

"

& T µ j &

$

( )λ λ β αµ β2 1 0− + − + =j

T 2n

$

SD

( ) ( )

( ) ( )( )( )

1 1 4

21

2 1 1 4 2

3 1 4 1

3 1 4 1

2

2

2

2

+ − ± + − −<

⇒ − < + − ± + − − <

− − + < ± + − − < − +

− − + < ± + − − < − +

β αµ β αµ β

β αµ β αµ β

β αµ β αµ β β αµ

β αµ β αµ β β αµ

j j

j j

j j j

j j j

$ ')(

( ) ( )( ) ( )

1 4 1

1 2 2 1

2

2 2+ − − < − +

⇒ + < − + −

⇒ < ⇒

β αµ β β αµ

β β αµ β β αµ

αµ α

j j

j j

j

- 2 - 2

0 > 0

$ '(

( ) ( )( ) ( ) ( ) ( )

( )

3 1 4

3 2 3 1 2 1 4

4 0

2 1

2 2

2 2

+ − > + − −

⇒ + − + > + − + −

⇒ − >

⇒ <+

β αµ β αµ β

β β αµ β β αµ β

β αµ

αβ

µ

j j

j j

j

j

8 + 8

$ j

( )0

2 1< <

+α

βA

$ A >>1 Q A

( ) ( ) ( ) ( )1 1 4

2

1 1

21

1

2

1

2

2+ − ± + − −

≅+ − ± −

= −

−

β αµ β αµ β β αµ βαµ β αµ

j j jj j;

SD

*

$

( )− < −

< ⇒ < + <

⇒ < < − ⇒ < <

1 11

21 0 1 2

12

1 1 1

αµµ

β

βµ

β

jj

j

A

A

- -

− < −

< ⇒ − < < +

⇒ + − < < ⇒ < <

11

21

1

21

1

21

11 1 1 1

β αµ αµ β αµ

β µ β β

j j j

jA

-

'! ! (

$&

( )r

rr

−+

≈ −1

11 1

24

/

$& NSD !

( )r

rr

−+

≈ −1

11 1

24/

SD !

( ! "

! '

! ! "

CG

CG

CG

SD

CG Conjugate Gradient CG

u v Q

v uTQ = 0 Q

vk kn

=−

01

n n× Q

Q

n

! "

a vk kk

n=∑

=

−0

0

1

! vjTQ # $

v a v a v v ajT

k kk

nk j

Tk

k

njQ Q

=

−

=

−∑ = ∑ = =

0

1

0

10

!

( ) ( ) ( )f x x x x x x x b x cT T T= − − = − +1

2

1

2* *Q Q

CG

%

!# $ vk kn

=−

01

x a v bk kk

n* = ∑ =

=

− −

0

1 1Q

$ #

!vjTQ

v x v b a v v a v vv b

v vjT

jT

k jT

kk

nj j

Tj j

jT

jT

jQ Q Q

Q* = = ∑ = ⇒ =

=

−

0

1 a

x0 0= $

!$

x xv b

v vv nk k

kT

kT

kk+ = + = −1 0 1 1

Q k , ,2, ... ,

x xn = *

Conjugate Directions &

"

vk kn

=−

01

Q

x0

( )α k

kT

k

kT

k

v x b

v v=

− −Q

Q k

x x vk k k k+ = +1 α

Qx bk − ≤ ε

n

Conjugate Directions - &

CG

&

x* n " %

x* ( )

x xv x b

v vvk k

kT

k

kT

kk+ = −

−1

Q

Q k

! vkTQ

[ ] [ ] [ ]v x x v x x v x b

x b

kT

k kT

k kT

kQ Q Q

Q

+

−

− = − − − =↑

=

1

1

0* *

*

vk Q x xk+ −1 * k

! v j j

k

=0 Q

( )

[ ] [ ][ ] [ ] [ ]

x x x xv x b

v vv

v x x v x x

v x x v x x v x x

k kkT

k

kT

kk

kT

k kT

k

kT

k kT

k kT

k

+

− + −

− + − − −

− = − −−

⇒ − = − =

⇒ − = − = − =

1

1 1 1

2 1 2 2 1

0

0

* *

* *

* * *

Q

Q

Q Q

Q Q Q

n

!

∀ ≤ ≤ − − =1 1 0j n x xjT

n v Q *

vk kn

=−

01

k $

!

[ ] [ ] ( )∀ ≤ ≤ − = − = − = ∇ =+ + + +1 01 1 1 1j k v x x v x b v e f x vjT

k jT

k jT

kT

k jQ Q*

CG

ek+1

vk+1 v j j

k

=0

n

CD


α k Q vk kn

=−

01

CD x x vk k k= + α

!

M x x vk j

k= = + ∈

=

x v subspace spanned by v j0 0,

!CD α k

( ) [ ] [ ]

( )

∂ α∂α

∂∂α

α α

α

f x vx v x x v x

f x v v e v

k kk k

Tk k

Tk k k k k

Tk

+= ⋅ + − + − =

= ∇ + = − =+

1

2

01

* *Q

α k α k

!

( )[ ] ( )e v x v b v

bv x v

v v

v x b

v vkT

k k k k k kk k

Tk

kT

k

kT

k

kT

k+ = + − = ⇒ = − =

− −1 0Q

Q

Q

Q

Qα α

CD

Mk

! k+1

CG

( ) ( )

x x v v v

f x v v vf x v

k k k

k k

j

Tk j

+

+

= + + + +

+ + + += ∇ =

1 0 0 0 1 1

0 0 0 1 11 0

β β β

∂ β β β∂β

where

k+1

Mk v j j

k

=0

k=n-1

Q

k v z0 0= z0

! zk+1 v j j

k

=0 Q z j j

k

=0

v z c vk k kj

jj

k+ + +

== + ∑1 1 1

0

!

∀ ≤ ≤ = = + =∑ +

⇒ = − ⇒ = − ∑

+ + +=

+ +

++

+ ++

=

0 01 1 10

1 1

11

1 11

0

m k v v v z c v v v z c v v

cv z

v vz

v z

v vv

mT

k mT

k kj

mT

jj

kmT

k km

mT

m

km m

Tk

mT

mk k

jT

k

jT

jj

j

k

Q Q Q Q Q

Q

Q

Q

Q v

k

n k

c n n( )−1 2

CG

"

Q CD

!

Q

Q Q

Q λ k kn

=1 θk kn

=1

!

( )θ θ θ λ θ λ θ θ λ δ

θ λ θ

kT

j kT

j j j kT

j j

j j j

k jQ

Q

= = = −↑=

CG

e b xk k= −Q

e b xk k= −Q k+1

!

v ev e

v vv x vk k

jT

k

jT

jj

j

kk k k k= − ∑ = +

=

−+

Q

Q0

11 ; x α

#CD $ CD α k

vk

CG

%

! CD

v ev e

v vv e

e e

e evk k

jT

k

jT

jj

j

kk

kT

k

kT

kk= − ∑ = +

=

−

− −−

Q

Q0

1

1 11

! ( )e e x x vk k k k k k+ +− = − =1 1Q Qα x x vk k k k+ = +1 α

( )v e e e ejT

kj

j jT

kQ = −+1

1α

∀ ≤ ≤ − ⊥0 1i k vk i e &

! ∀ ≤ ≤ − ⊥0 1i k ek i e span v span ei ik

i ik

=−

=−=0

101

( )v e e e e

e e j k

j kjT

kj

j jT

kk

kT

k

Q = − =

= −

≤ < −

+−1

11

0 0 11

1

α

α

!

v ev e

v vv e

e e

v vvk k

jT

k

jT

jj

j

kk

kT

k

k kT

kk= − ∑ = −

=

−

− − −−

Q

Q Q0

1

1 1 11α

! ( )e e x x vk k k k k k+ +− = − =1 1Q Qα

( )v v e e v e vjT

jj

j jT

jj

jT

jQ = − = −+

1 11α α

!

v v e v e ev e

v vv e ek

Tk

kkT

kk

kT

kjT

k

jT

jj

j

k

kkT

k− −−

− −−

− −=

−

−− −= − = − − ∑

= −1 1

11 1

11 1

0

2

11 1

1 1 1Q

Q

Qα α α

CG

v ee e

e evk k

kT

k

kT

kk= +

− −−

1 11

Mk−1 ek

Q

ek =0

ek ≠ 0

∀ ≤ ≤ − ⊥0 1i k ek i e ∀ ≤ ≤ − ⊥0 1i k vk i e

ek span v span ei ik

i ik

=−

=−=0

101

! CG

" PARTAN

#

x0

v e b x0 0 0= = −Q

βkkT

k

kT

k

e e

e e=

− −1 1 k

v e vk k k k= + −β 1

α kkT

k

kT

k

v e

v v=

Q


Qx bk − ≤ ε

n

Conjugate Directions - !

NSD

CG

$

CG

NSD CG

CG

CG NSD

n

n

CG

Q v Rn≠ ∈0 ; v

v v v v vk jj

k, , , , Q Q Q Q2 1

0

1

−=

−= % k

& &

n Q v Rn≠ ∈0 ; v

v v v v vk jj

k, , , , Q Q Q Q2 1

0

1

−=

−=

k n≤

# "

α αjj

j

kj

j

j

kv vQ Q

=

−

=

−∑ = ∑

=0

1

0

10

v v k Q

Q n

CG

'

# "

e j j

k

=0 v j j

k

=0

CG

span e e span e e e e

span v v

kk

k k

0 1 2 0 02

0 0

0 1 2

, , , , , , , ,

, , , ,

e e

v v

=

= =

Q Q Q

M

span e span e0 0= k=0

k % span v span e0 0= v e0 0= CG

k+1

[ ]e b x b x v e vk k k k k k k k+ += − = − + = −1 1Q Q Qα α

Q Mvk k∈ +1 ek k∈ M & ek+1

ek k+ +∈1 1M

vk k+ +∈1 1M v e vk k k k+ + += +1 1 1β CG

CG ( %

#n "

n

&

&

CG

x

x

1 0 00

0

2 0 10

0 11

1

3 0 20

0 21

1 22

2

1 00

01

12

2

= +

= + +

= + + +

= + + + + ++

x e

x e e

x x e e e

x x e e e ek k k k kk

k

γ

γ γ

γ γ γ

γ γ γ γ

span e e span e e e e

span v v

kk

k k

0 1 2 0 02

0 0

0 1 2

, , , , , , , ,

, , , ,

e e

v v

=

= =

Q Q Q

M

( ) ( )( ) ( )

( )

x x P e x P e

x x P e x P e

where

P

k k

k k k k kk k

1 0 0 0 2 0 1 0

3 0 2 0 1 0 0

0 1 2 2

= + = +

= + = +

= + + + +

+

Q Q

Q Q

Q I Q Q Q

x

x

:

γ γ γ γ

k

!

" Mk k #$

#CD $ CG

%

# & $

P A A A P A( ) ( )⋅ = ⋅

'

CG

(

( )( ) ( ) ( )[ ]( )

( ) ( ) ( )

( ) ( )[ ] ( )

x x x x P e

x x P x x I P x x

f x x x x x

x x I P x x

k k

k k

kT

k

Tk

+

+ +

− = − + =

= − + − = + −

= − − =

= − + −

1 0 0

0 0 0

1 1

02

0

1

21

2

* *

* * *

* *

* *

Q

Q Q Q Q

Q

Q Q Q

k e x b0 0= −Q

& " "k+1 &

' ' #n $ '

CG & xk+1 #CG $

"Mk ( ) ( ) ( )f x x x x xT= − −1

2* *Q

'

( )

( ) ( )[ ] ( )f x x x I P x xkT

k

kj

j

k+ = − + −

=

1 02

0

0

1

2min * *

γQ Q Q

( )P xk γ kj

j

k

=0

)

CG

CG "' Q λ j j

n

=1

( )

( )[ ] ( )f x P f xk j k jj j

n+ ≤ +

=

12

0

1

1maxλ

λ λ

k ( )P xk

CG

( )x x0 − * θ j j

n

=1 Q

( )x x j jj

n0

1− = ∑

=* α θ

& " "'

( )

( ) ( )[ ] ( )

( )[ ]

f x x x I P x x

P

kT

k

j jT

j

nT

j j jT

j

nj k j j j

T

j

nj j

j

n

kj

j

k+

= = = =

= − + −

=

= ∑

∑ +∑ ∑

=

=

1 02

0

1 1

2

1 1

0

1

2

1

21

min * *γ

α θ λ θ θ λ λ θ θ α θ

Q Q Q

( )[ ] ( )[ ] = ∑

⋅ +∑

= +∑

= = =

1

21

1

21

1

2

1

2

1

2α λ θ α λ λ θ λ λ α λj jj

njT

j j k j jj

nj k j

j

nj jP P

k=0 Q = ∑=

λ θ θj j jT

j

n

1"&

( ) ( ) ( )f x x x x xTj j

j

n0 0 0

2

1

1

2= − − = ∑

=* * Q α λ

( ) ( )[ ] ( )[ ] ( ) f xk+1 = +∑ ≤ +

=

1

21 1

2

1

2 20λ λ α λ λ λ

λj k j

j

nj j j k jP P f x

j

max

"[a,b] n-k Q

x0 "' "b

CG

( ) ( )f xb a

b af xk+ ≤ −

+

1

2

0

λ j j

n

=1 !

( )12

21

1

2

2+ =

+⋅ + −

⋅ − ⋅ − −λ λ λ λ λ

λλ λ

λλ λ

λP

a b

a bk

k

k

[a,b] ! " !

λ j j

k

=1 #

# b

( )

( )[ ] ( ) ( )f x P f xa b

a bf xk j k j

a bj j

n+≤ ≤

≤ + ≤+

⋅ + −

=

12

0

2 2

0

1

12

2max maxλ λ

λ λ λ

"

λ λλ

λ λλ

λ λλ

1

1

2

21

− ⋅ − − ≤k

k

" ! #

( ) ( ) ( )f xa b

a bf x

b a

b af xk

a b+

≤ ≤≤

+

⋅ + −

= −

+

1

2 2

0

2

02

2max

λλ

! !

#

CG

[ ]100 100− +ε ε, Q !

$n=100, k=5% ! [ ]1 1− +ε ε,

( ) ( ) ( )( ) ( ) ( ) ( )f x f x f x6

2

02

01 1

1 1≤ + − −

+ + −

= ⋅ε ε

ε εε

( ) ( ) ( )( ) ( ) ( ) ( ) ( )f x f x f x f x6

12

012

0 0100 1

100 10 98 0 79≤ + − −

+ + −

≅ ⋅ ≅ ⋅ε ε

ε ε. . #

NSD

[ ]100 100− +ε ε, Q !

$n=100, k=90% !

( ) ( ) ( )( ) ( ) ( ) ( )f x f x f x6

2

02

0100 1

100 10 98≤ + − −

+ + −

≅ ⋅ε ε

ε ε.

NSD ( ) ( ) ( )( ) ( ) ( ) ( )f x f x f x6

91 2

0 0100 1

100 10 026≤ + − −

+ + −

≅ ⋅

⋅ε εε ε

. #

& # #

NSD ! # CG

" !

λ λ λ λ1 2 3≥ ≥ ≥ ≥ n Q

" ! !

( ) ( )f x f xn

n1

1

1

2

0≤ −+

λ λλ λ

!

( ) ( )f x f xn

n2

2

2

2

0≤ −+

λ λλ λ

CG

Q ! ! #

# Condition #

!

# a=b !

' # CG # (

! n-m+1 m ! ! )

[a,a] [a,b] ! # #

# #

CG

#

# CG

m<<n ! [a,b]

CG ! m CG ! n * !

!

! mL # * ! " CG

( ) ( ) ( )f xb a

b af x

b a

b af xmL mL L

L

+ −≤ −+

≤ ≤ −

+

1

2 2

0

b/a ( ## #

λ λmax min $ # %

NSD NSD

( ) ( ) ( )f x f x f xmL

mL m L

+ ≤−+

=

−+

1

2

0

2

0λ λλ λ

λ λλ λ

max min

max min

max min

max min

λ λmax min =1000

# b/a=10 m=4

CG

(

CG # (

( ) ( ) ( )

( ) ( ) ( )

PCG x f x f x

NSD x f x f x

mL

LL

mL

LL

: .

: .

f

f

+

+

≤

=

≤

=

1

2

0 0

1

4 2

0 0

9

110 67

999

10010 992

PCG

N # H Y

LS Y HX N= + #

[ ] [ ] argmin argminX Y HX Y HX X H HX X H Y Y YX

T

X

T T T T T= − − = − +2

# b H YT Q H HT

!

' H HT! #

# $ ! %

' ' '

!! #

[ ] [ ] ( ) argminX Y HX Y HX XX

T T= − − + ⋅

β 12

Q ! ++ ! !

β H HT # ! Q = + ⋅ ⋅H HT Tβ 1 1 !

! $H HT % β

Interlock ! # !% β Q

! n-m # m

CG

Condition # NSD

CG PCG Q

CG

f(x)

CG

!

n " n

# SD !

$

CG

# %

α k & # α k

Fletcher-Reeves

βk Fletcher-Reeves

$ Q

Q

' ! ! !

βkkT

k k

kT

k

e e e

e e=

− −

− −

1

1 1

CG

(

Polak-Ribiere

x0

( )v e f x0 0 0= = −∇

( )e f xk k= −∇ k

βkkT

k

kT

k

e e

e e=

− −1 1 ek

v e vk k k k= + −β 1

( )F xk

( )α k

kT

k

kT

k k

v e

v x v=

F


x xn0 = n

! " #

ek ≤ ε $

Conjugate Directions - #

k<n CG

n PCG

CD

n CG

CG

Q

CG

)

CG

CG #(

$ !

CD

CD

CD Q

Q & CG

CD

CG PCG

PCG PCG CG

CG

PCG CG

SD CG

NSD

SD ! CG

NSD !

!

"

" "

!

n

F !

e e1 2, f(x) x x1 2,

( )e b x f x1 2 1 2 1 2, , ,= − = −∇F

#

( ) ( ) [ ] [ ] [ ] ( ) ( ) [ ] [ ] [ ]

f x f x e x x x x x x o x x

f x f x e x x x x x x o x x

T T

T T

2 1 1 2 1 2 1 2 1 2 12

1 2 2 1 2 1 2 1 2 2 12

1

21

2

− = − − + − − + −

− = − − + − − + −

F

F

#

− − ≅ −e e x x2 1 2 1F

$

! ! e b x1 2 1 2, ,= −F % %

! !

n x ek k kn, =0 n+1

# !

− − ≅ − =− −e e x x k nk k k k1 1 1 3F ,2, ,.....,

[ ] [ ] x x e ek k k k kn− −− − =1 1 0

, E X

" EX− =1 F

n+1 F x ek k kn, =0

x ek k kn, =0 " ! F

! F

#

F

H0 ! n-1 & j q pj j= F p qj j j

n,

=−0

1

#

H HH H

Hk k

k k k k k kT

kT

k k k

p q p q

q p q+ = +

− −−1

" k & j p qj k j= +H 1

n H Fn = −1 p qj j j

n,

=−0

1

F

! ! H0

k+1 ∀ ≤ ≤ − =0 1j k qj k j p H k

# 0 1≤ ≤ −j k j=k

'

∀ ≤ ≤ − = +−

−−+0 1 1j k q q

p q

q p qp q q qk j k j

k k k

kT

k k kkT

j kT

k j H HH

HH

Hk H0 %

H0 Hk "

# Hk j jq p=

∀ ≤ ≤ − = +−

−−+0 1 1j k q p

p q

q p qp q q pk j j

k k k

kT

k k kkT

j kT

j HH

H

# %

[ ] [ ]p q q p p p q p p q p p q pkT

j kT

j

q p

kT

j kT

j kT

kT

j

T

k kT

j

j j

− = − = − =↑

=

− =↑=F

F F

F F

F 0

∀ ≤ ≤ − =+0 1 1j k q pk j j H

# j=k

H HH H

HH Hk k k k

k k k k k kT

kT

k k kk k k k k k kq q

p q p q

q p qq q p q p+ = +

− −−

= + − =1

∀ ≤ ≤ − =0 1j n q pn j j H j=k

# "

( )

H H

H F

n n n n

n

q q p p Q P

PQ QP

0 1 0 1

1 1 1 1

− −

− − − −

=

⇒ =

⇒ = = =

!

x x ek k k k k+ += +1 1α H

x x ek k k k= + +α H 1 α k

( ) ( ) ( )

( )( )

∂∂α

∂∂α

α α

α

α

f xx e x x e x

x e x

e x x

e e

e e

e e

k

k kk k k k

Tk k k k

kT

k k k k k

kkT

k k

kT

k k k

kT

k k

kT

k k k

++ +

+ +

+

+ +

+

+ +

= + − + − =

⇒ + − =

⇒ =−

=

11 1

1 1

1

1 1

1

1 1

0

0

H F H

H F H

H F

H FH

H

H FH

* *

*

*

e

! ! F

!

"

# $ q p qkT

k k k− <H 0

! Hk

q p qkT

k k k− H %

x0

H0

( )e f xk k= ∇ k

α k

( )f x ek k k+ α H

x x ek k k k k+ = +1 α H

H HH H

Hk k

k k k k k kT

kT

k k k

p q p q

q p q+ = +

− −−1

q e e p ek k k k k k k= − =+1 , α H

ε ! ( )e f xk k= ∇ ≤ ε "

#$ %

DFP

Powel Fletcher !Davidon

& ! &

!

! & CG

% ! !%

DFP

Hk

!

!DFP

%

H HH H

Hk k

k kT

kT

k

k k kT

k

kT

k k

p p

q p

q q

q q+ = + −1

!

x

x x x x xp p

q px x

q q

q qxT

kT

kT k k

T

kT

k

T k k kT

k

kT

k kH H

H H

H+ = + − >1 0

x0

H0

( )e f xk k= ∇ k

α k

( )f x ek k k+ α H

x x ek k k k k+ = +1 α H

H HH H

Hk k

k kT

kT

k

k k kT

k

kT

k k

p p

q p

q q

q q+ = + −1

q e e p ek k k k k k k= − =+1 , α H

ε ! ( )e f xk k= ∇ ≤ ε "

#$ %

DFP %

H0 k=0

Hk

H H Hk k k= ⋅1 2 1 2/ /

a x qk k k= =H H1 2 1 2/ / b

x

x x a ap x

q pa

bb

b ba

a aa b

b b

p x

q p

a a b b a b

b b

p x

q p

Tk

T kT

kT

k

TT

T

TT

TkT

kT

k

T T T

TkT

kT

k

H + = + − =

= − + =−

+

1

2

2 2 2 2

q e e p ek k k k k k k= − =+1 , α H DFP

q p e e e e ekT

k k k k k k k kT

k k= − =+α α1T H H

e ekT

k k+ =1 0H

x xa a b b a b

b b

p x

e eT

k

T T T

TkT

k kT

k kH

H+ =

−+ ≥1

2 2

0α

a b a b⋅ ≥ , !

" "

x qk= β b a

x xp q

e e

e e

e ee eT

kkT

k

k kT

k k

k kT

k k

k kT

k kk k

Tk kH

H

H

HH+ = = = >1

2 2 2 2

2 0β

α

β α

αβ α

x xTkH + >1 0

F DFP " "

Hk j jq p k= = − for j 0 1 1, , ,

#

F " " " "

k DFP p qj j j

n,

=−0

1

∀ ≤ < ≤ =0 0i j k piT

j p F F p j j

k

=0

H0 0> $

H HH H

Hk k

k kT

kT

k

k k kT

k

kT

k k

p p

q p

q q

q q+ = + −1

k 0 j p qj k j= +H 1

p qj j j

n,

=−0

1

F n H Fn = −1

q e e b x b x x x pk k k k k k k k= − = − − − = − =+ + +1 1 1F F F F

H HH H

H

H

HHk k k k

k kT

kT

kk

k k kT

k

kT

k kk

kT

k k

kT

k kk k k kq q

p p

q pq

q q

q qq

q q

q qq p p+ = + − = −

+ =1 1

k=0 "

$ F p0

" p q0 1 0= H

∀ ≤ < ≤ − =0 1 0i j k p piT

j F k-1 % "

0 2≤ ≤ −i k ∀ ≤ ≤ − =0 1j k qj k j p H

e e q e p

e e e p e p p

k k k k k

k k k k i k i

= − = − =

= − + − = = − + +− − −

− − − − + − +

1 1 1

2 2 1 1 1 1 1

F

F F

&

0 2≤ ≤ −i k pi %"

p e p e p p p p eiT

k iT

i iT

k i iT

i= − + + = =+ − + +1 1 1 1 0F

k-1 " " p eiT

i+ =1 0 " "

" p ekT

k− =1 0

H H Fk kq p pi i i= =

[ ]01 1= = = = =

↑=

−p e p e p e p p p piT

k iT

k iT

k

p ek

iT

k k iT

k

k k k k

H F FH FH H F

H

k k k

αα

0 1≤ ≤ −i k " F "

Hk+1

H HH H

H

H

HFk i k i

k kT

kT

ki

k k kT

k

kT

k ki i

k

kT

k

k k

kT

k kkT

i iq qp p

q pq

q q

q qq p

p

q p

q

q qp p p+ = + − = + −

=1

i=k " ∀ ≤ ≤ − =+0 1 1j k q pk j j H

" ∀ ≤ ≤ − =0 1j n q pn j j H

( )

H H

H F

n n n n

n

q q p p Q P

PQ QP

0 1 0 1

1 1 1 1

− −

− − − −

=

⇒ =

⇒ = = =

n

CG

F DFP

CG

CG H0 = I

! k

∀ ≤ ≤ − = =↑=

↑=

0 1i k p q pk i

p q

k i

q p

i

i i k i i

: H F H

F H

H Fk pi ik=−01

n H Fk

" #

Broyden

0 ≤ ≤ =i k q pi iF

0 1≤ ≤ =+i k q pk i iH Hk

Hk

0 1≤ ≤ =+i k p qk i iB " # Bk

p qi i,

"DFP # $

! DFP

H HH H

HB B

B B

Hk k

k kT

kT

k

k k kT

k

kT

k kk k

k kT

kT

k

k k kT

k

kT

k k

p p

q p

q q

q q

q q

p q

p p

p p+ += + − ⇔ = + −1 1

%

&

" #

' '

!

a,b A " #

!

A ab AA ab A

b A aT

T

T+ = −+

− −− −

−1 1

1 1

11

! A abT+

[ ] [ ]

( )

I A ab A A abA ab A

b A a

I ab Aab A ab A ab A

b A a

I ab AI ab A

b A aab A

I ab Ab A a

b A aab A I

T TT

T

TT T T

T

TT

TT

TT

TT

= + − ++

=

= + − ++

=

= + − ++

=

= + −+

+=

−− −

−

−− − −

−

−−

−−

−−

−−

11 1

1

11 1 1

1

11

11

11

11

1

1

1

1

1

b A aT −1

( )ab A ab A b A a ab AT T T T− − − −= ⋅1 1 1 1

# ' Bk

! "

H B HH H H

knew

k knew k

Tknew

k

kT

k

k kT

kT

k

k kT

knew

knew

k kT

kT

k

q q

q p

p p

q p

p q q p

q p+ +

−= = + +

− +

1 11 1

$ " # DFP

Broyden, Fletcher, Goldfarb & Shanno

BFGS

(

DFP " #

F p qj j j

n,

=−0

1

H Fknew

DFP

Broyden BFGS DFP

!

( )0 1 11 1 1≤ ≤ = + −+ + +φ φ φ H H HkB

kDFP

kBFGS

!

H HH H

H

H HH H H

kDFP

kDFP k k

T

kT

k

kDFP

k kT

kDFP

kT

kDFP

k

kBFGS

kBFGS k

TkBFGS

k

kT

k

k kT

kT

k

k kT

kBFGS

kBFGS

k kT

kT

k

p p

q p

q q

q q

q q

q p

p p

q p

p q q p

q p

+

+

= + −

= + +

− +

1

11

!

[ ]

H H

HH

H

kB

kDFP

k kT

k kT

kDFP

kk

kT

k

kDFP

k

kT

kDFP

k

v v

where q qp

p q

q

q q

+ = +

= ⋅ −

1

0 5

φ

:.

v

H FkB F p qj j j

n,

=−0

1

φ

φ

Broyden

m<n BFGS DFP

CG

Broyden

!

( ) ( ) ( )f x x x x xT= − −1

2* *F x*

"

x x ek k k k k+ = +1 α S

α k ( )e x xe e

e ek k k

kT

k k

kT

k k k= − =F

S

S FS* ; α "

"

( )( )

f x

f x

B b

B bk

k

k k

k k

+ ≤ −+

1

2

S Fk ! Bk bk

! #$ SD # !

" %

( ) ( )( )

[ ][ ][ ]

f x f x

f x

e e

e e e e

k k

k

kT

k k

kT

k k k kT

k

−=+

−1

2

1

S

S FS F

" T S FS Sk k k k k ke= =1 2 1 2 1 2/ / / ; v %

$&

( ) ( )( )

[ ][ ][ ]

f x f x

f x

v v

v v v v

k k

k

kT

k

kT

k k kT

k k

−=+

−1

2

1T T

Tk S T S S Fk k k k1 2 1 2/ /− = T S FSk k k= 1 2 1 2/ /

!

# ' % ( S Fk

" ! !

# NSD ! Sk I= 1

F !

!

S Fk ! S Fk = −1 2

#

S H Fk k k n= →

→−1 3

k k

" ! S Fk

∀ ≤ ≤ − = =0 1i k p q pk i k i i S F S

k ( S Fk ∀ ≤ ≤ −0 1i k i p

( )

" !

n n× A Interlock

! n a λ λ λ λ1 2 3≤ ≤ ≤ ≤ n

" µ µ µ µ1 2 3≤ ≤ ≤ ≤ n A aaT+

λ µ λ µ λ µ µ λ µ1 1 2 2 3 3≤ ≤ ≤ ≤ ≤ ≤ ≤ ≤ ≤ n-1 n n

"

$

F ! (

* Hk !

( ( !

(

NSD

[1e5,1.001e5] n F ! "

NSD SD ( Condition Number -

( DFP

n-1 ( (

Condition Number DFP !

! scale

A αA !

DFP ! ! α

! γk Hk k

"

H HH H

Hk k k

k kT

kT

kk

k k kT

k

kT

k k

p p

q p

q q

q q+ = + −1 γ γ

H Fk ( ! (

" (

λγ

λmin maxH F H Fkk

k≤ ≤1

"

" x A

λ λmin maxAx Ax

x xA

T

T≤ ≤

λ λmin maxA Minx Ax

x xA Max

x Ax

x xx

T

T x

T

T= = ;

λ λmin maxH FH F

H Fk

Tk

T kx x

x x≤ ≤

x pk=

p p

p p

p q

p p

p p

p qkT

k k

kT

k

kT

k k

kT

k

kT

k

kT

k k

H F H

H= ⇒ = kγ

γk

F

Broyden

Hk+1 Hk

BFGS Hk I=

Hknew k

Tk

kT

k

k kT

kT

k

k kT

k kT

kT

kI

q q

q p

p p

q p

p q q p

q p+ = + +

− +

11

Polak & CG

Rebiere

!

n

" CG CG

#

LS

LS

Least-Squares

( ) ( )[ ] ( ) ( ) ( )[ ] ( )( )

( )

( ) ( )f X r X r X r X r X r X

r X

r X

R X R Xkk

mm

m

T= ∑ =

= ⋅=

2

11 2 1

2

n X ( ) r Xk km

=1

!" !! # "

yk km

=1 $

# X

x yk k km, =1 $

n

( ) ( )[ ]ε20 1 0

01

1 2

1a a a y a x a x a xn k k k n k

n

k

m, , , = − + + +∑

=

( )k m r x x xk n= =1 01 2, , , , , $ !

%

( ) ( )f x x x r x x xn k nk

m1 2

21 2

1, , , , , , = ∑

=

LS

#

LS

l2 $

n $

n

Maximum Likelihood "

$

& ML ML

LS '

X N

$

!

( )p nn= −

1

2 2

2

πexp

p(n) n

$ θ y y X nT= +θ %

m

θ1

θNn

X1

XN

Y

LS

(

θ yk km

=1 Xk km

=1

( ) [ ] p nn y X

p y Xkk k k

T

k k= −

= −

−

=1

2 2

1

2 2

22

π π

θθexp exp , /

m

[ ] [ ]p y Xy X

C y Xk k km k k

T

k

mk k

T

k

m, exp exp= = =

= −−

∏ = ⋅ − −∑

1

2

1

2

1

1

2 2

1

2θ

π

θθ

% θ

[ ] argmax , argminθ θ θθ θ

ML k k km

k kT

k

mp y X y X= = −∑

=

=1

2

1

LS

LS

LS

LS

% LS

( ) ( )[ ] ( ) ( ) ( )[ ] ( )( )

( )


r X

r X

R X R Xkk

mm

m

T= ∑ =

= ⋅=

2

11 2 1

2

LS

LS

( ) ( )r X y h X X Y HXk k kT= − ⇒ = − R

m n× H

( ) [ ] [ ]∂∂f X

XH Y HX H H X H YT T T= = − − ⇒ =0 2

H HT

m n× H

H !" # m n≥

m n= !

" # H

$

! H HT Pseudo-Inverse

X H H H Y H YT T= =−1 #

H H#

H H H H H H IT Tn n

# = =−

×1

1

HH H H H H IT Tm m

# = ≠−

×1

2

! n

!m>n ! m m×

H H H H H H H HT T T T# = = =− − − −1 1 1

m n= 3

LS

%

HH H H H HT T# =−1

4

HH HHk# #=

HH H H H HT T# =−1

5

X H H H Y H YT T= =−1 #

( ) [ ] [ ] [ ] [ ][ ] [ ]

f X Y HX Y HX Y HH Y Y HH Y

Y I HH HH HH HH Y Y I HH Y

T T

T T

= − − = − − =

= − − + = −

# #

# # # # #

LS

!CG !NSD !SD

"n=200 # !

200 400< <n

! n n×

CG n=400

Gauss-Newton LS

LS

( ) ( )[ ] ( ) ( ) ( )[ ] ( )( )

( )


r X

r X

R X R Xkk

mm

m

T= ∑ =

= ⋅=

2

11 2 1

2

( ) ( ) ( ) ( ) ( ) ( ) ( ) ( ) ( )∂∂

∂∂

∂∂

∂∂

∂∂

f X

X

r X

Xr X

r X

X

r X

X

r X

XR X A X R Xk

kk

mm= ∑ =

==

2 2 21

1 2

LS

&'

"# n m× A(X)

( ) ( ) ( ) ( ) ( )

( ) ( ) ( ) ( )

∂∂

∂∂

∂∂

∂∂

∂∂

2

21

2

21

2

21

2 2

2 2

f X

X

r X

X

r X

X

r X

Xr X

A X A Xr X

Xr X

k

k

mk

Tk

kk

m

T kk

k

m

= ∑ + ∑ =

= + ∑

= =

=

LS

!

"# ! ( )r Xk

( ) ( ) ( ) ( ) ( ) ( ) ( )∂∂

∂∂

2

2

2

21

2 2 2f X

XA X A X

r X

Xr X A X A XT kk

k

m T= + ≅∑=

" ( )r X y H Xk k k= − # "(# ( )r Xk

!

( )r Xk

n !

!

( )r Xk LS

!")#

" #

( )r Xk !

LS

LS

Gauss-Newton

( ) ( )[ ] ( ) ( ) ( )[ ] ( )X X A X A X A X R X X A X R Xk k kT

k k k kT

k k+−

= − = −11 #

! "

X0

k

A(X)

d

( ) ( ) ( ) ( )A X A X d A X R XkT

k k k k= −

X X dk k k+ = +1

ε dk ≤ ε

!

LS Gauss-Newton

Gauss- X* # LS

X* Newton

( ) ( )[ ] ( ) ( ) X X A X A X r Xr X

XX X O X Xk k

k

k

mk k+

−

=− ≤ ⋅ ∑

⋅ − + −11

2

21

2* * * **

* *∂

∂

LS

( ) ( )[ ] ( ) ( )X X X X A X A X A X R Xk k kT

k k k+−

− = − −11

* *

X*

( ) ( )[ ] ( ) ( ) ( ) ( )[ ]( )[ ]X X A X A X A X R X A X A X X Xk kT

k kT

k k− = + −−

* * * *1

$

( ) ( )[ ] ( ) ( ) ( ) ( )[ ]( )[ ]( ) ( )[ ] ( ) ( )

( ) ( )[ ] ( ) ( ) ( ) ( ) ( ) ( )[ ]( )[ ]

X X A X A X A X R X A X A X X X

A X A X A X R X

A X A X A X R X A X R X A X A X X X

k kT

k kT

k k

kT

k k k

kT

k k k kT

k k

+−

−

−

− = + − −

− =

= − + −

11

1

1

* * * *

* * *

( ) ( ) ( ) ( ) ( ) ( )[ ]( )

( ) ( ) ( ) ( ) ( ) A X R X A X R X A X A X X X

r Xr X

XX X O X X X X

k k kT

k k

k kk k

k

mk k

Tk

* * *

* * *

= + − +

+ ∑

− + − −=

∂

∂

2

21

( ) ( )[ ] ( ) ( ) ( )

( ) ( )[ ] ( ) ( )

X X A X A X r Xr X

XX X

A X A X O X X X X

k kT

k k kk k

k

mk

kT

k kT

k

+−

=

−

− = ∑

− +

+ − −

11 2

21

1

* *

* *

∂∂

$ Xk X*

( ) ( )[ ] ( ) ( ) X X A X A X r Xr X

XX X O X Xk

Tk

k

k

mk k+

−

=− ≤ ∑ − + −1

1 2

21

2* * * **

* *∂

∂

LS

%

1

n n LS 2

# ( ) r Xk kn* = =0 1

3

# X Xk − *

& Gauss-Newton 4

' '

' Gauss-Newton

-Gauss

dk ' Newton

%

X0

k

A(X)

d

( ) ( ) ( ) ( )A X A X d A X R XkT

k k k k= −

( )f X dk k+ α α

X X dk k k k+ = +1 α

ε dk ≤ ε "

!

LS Gauss-Newton %

LS

LS

LS

!BFGS" #

Levenbeg-Marquardt

$

( ) ( )A X A X IkT

k + λ

λ

LS

( ) ( )[ ] ( )[ ]f X Y S X Y S XT= − −

S(X)

$ ( )Z G X=

( ) ( )( )S X S G Z HZ= =−1

X Z Z X

$ X

( ) ( )[ ] ( )[ ] [ ] [ ]

[ ]f X Y S X Y S X Y HZ Y HZ

Z H H H Y

T T

optT T

= − − = − −

⇒ =−1

LS

G " ( )Z G X= % X

!

x yk k km, =1

A f xkcos 2 0π φ+

$

( ) ( )[ ]ε φ π φ20

2

12A y A f xk k

k

m, cos= − +∑

=

$ LS

( ) ( ) ( ) ( )A f x A f x A f xk k kcos cos cos sin sin2 2 20 0 0π φ φ π φ π+ = −

( ) ( )α φ α φ1 2= =A Acos sin ,

$

( ) ( ) ( )[ ]ε α α α π α π21 2 1 0 2 0

2

12 2, cos sin= − +∑

=y f x f xk k k

k

m

$

[ ]αα

1

2

= H H H YT T

( ) ( )( ) ( )

( ) ( )

H

f x f x

f x f x

f x f x

y

y

ym m m

=

−−

−

=

cos sin

cos sin

cos sin

2 2

2 2

2 2

0 1 0 1

0 2 0 2

0 0

1

2

π ππ π

π π

Y $

LS

&

A tg= + =

−α α φαα1

222 1 1

2, $

S(X) '

X β α ( ) ( )S X H= ⋅α β

H !"

α

$ β

( ) ( )[ ] ( )[ ]( ) ( ) ( )[ ] ( )

f Y H Y H

H H H Y

T

optT T

α β α β α β

β α α α α

, = − −

⇒ =

$

( )( ) ( ) ( )[ ] ( )[ ]f Y I H H H YoptT T Tα β α α α α, = −

$

( )( ) ( ) ( )[ ] ( )f Y H H H YoptT T Tα β α α α α, =

g(x) ( )ψ α αk

kxx e K, ...= =− k 1 0

$

( ) ( ) g x x kxk kk

K

kk

K= ∑ = −∑

= =β ψ α β α, exp1 1

0 0

$ LS %

( ) ( ) ε α β β β α21

11

2

0

0, , exp ... , k j k j

k

K

j

mg x kx= − −∑

∑

==

LS

m x

βk

( )( )

( )( ) ( )[ ] ( )β α

β α

β αα α α=

=1

m

T TH H H Y

( )( )

( )

H

x x K x

x x K x

x x K x

g x

g x

g xm m m m

=

− − −− − −

− − −

=

exp exp exp

exp exp exp

exp exp exp

α α αα α α

α α α

1 1 0 1

2 2 0 2

0

1

2

2

2

2

Y

( ) ( ) ( )[ ] ( )f Y H H H YT T Tα α α α= α

LS

LS

! n "

LS # n

( ) ( ) ( ) ( ) ( ) ( )r x x x X r X R X R Xk n kn

kk

n T1 2 1

2

10, , ... , R X = 0 f= ⇔ ⇔ = ∑ == =

Xk ""

( ) ( ) ( )( )R X R X A X X XkT

k k= + −

R(X) " A

LS

$

( ) ( )X X A X R Xk kT

k k+−= −1

Gauss-Newton

( )[ ] ( )X X A X R Xk kT

k k+ = −1#

A m=n

Gauss-Newton

Newton-Raphson

x f(x)

x0 Newton-Raphson f(x)=0

"

( ) ( ) [ ] ( )m x f x x x f x= ⋅ − +' 0 0 0

x

( ) ( ) [ ] ( ) ( )( )m x f x x x f x x

f x

f x= ⋅ − + = ⇒ = −'

'0 0 0 1 00

00 x

" %

& Newton-Raphson

Newton-Raphson &

f(x)

LS

'

!

A " "

# %

Newton-Raphson

R(X)

( ) ( )R X X X= − =Φ 0

( % ( )X X* *= Φ

X0

( )X Xk k+ =1 Φ k

( )X Xk k+ − ≤1 Φ ε

ε

(

LS

( ) ( )r x x x= + −1 2exp

x x xk k k k+ += − = +1 12 11

21exp log x

Newton-Raphson

( )( )

( ) ( )( )x x

x x

x

x x

xk kk k

k

k k

k+ = −

+ −−

=− −

−11 2

1 2 2

1 2 2 1

1 2 2

exp

exp

exp

exp

x0 1=

x xk k+ = −1 2 1exp x xk k+ = +11

21log

Newton-Raphson

6.3890.34660.6089

3.5e50.14880.3014

!"0.06930.1033

#0.03350.0168

$0.01650.0005

0.00821e-7

$ "

% R %RX=P

R & U

D % %'

(

LS

R (

U L D X P+ + =

"

Jacoby ( )[ ]X D P U L Xk k+−= − +1

1

'GS& Gauss-Siedel X D L P UXk k+−= + −1

1

( %" R

" RX=P

( )ε2 1

2X X RX P X CT T= − +

( DNSD

X X H P RXk k k+−= + −1

1

% " H

DNSD H=D %R

( )[ ] [ ]X X D P U L D X D P U L Xk k k k+− −= + − + + = − +1

1 1 ( )

" DNSD Jacoby "

"

R = U + L + D

LS

X D L P UXk k+−= + −1

1 GS " "

" " "

" %

D L X P UXk k+ = −+1

%Xk+1 1( ) Xk+1 2( ) " Xk+1 1( )

" "

$ " )

" " "

" ) " "

R(X) " % ( ) ( )R X X X= − =Φ 0

Jacoby " X ""

" R

% ' &

( )x x= Φ "" % "

( )Φ x x

( )x xk k+ =1 Φ

'" & * + %

"" +

y=x

( )y x= Φ

x0

LS

( ) exp log1 2 11

211 1 x ; (2) xk k k kx x+ += − = +

! "

y=x

( )y x= −exp 2 1

x0

x0

y=x

( )y x= Φ

x0

LS

#

x* , .= −0 0 8 $ ! "

$ x0

x* .= −0 8 %$ $

0 5 1. log( )x + &!

' $ '$ y=x %$ % exp( )2 1x +

% %$ $ x0 x* , .= −0 0 8

% -0.8! %$

&!

%$

%$ x* = 0 x* .= −0 8

' $ %$

' %$ Φ( )x

y=x ( )y x= +0 5 1. log

x0x0

LS

( ) ( )R X X X= − =Φ 0 $ R(X)=0 $ '

% ( )∇Φ <X * 1 $ ( )Φ X

%$ ( )X Xk k+ =1 Φ X*

'$ %

( ) ( ) ( )( ) X X X X X X o X Xk k k k+ = = + ∇Φ − + −1 Φ Φ * * * *

' ( )X X* *= Φ

( ) X X X X X o X Xk k k+ − = ∇Φ ⋅ − + −1 * * * *

( )∇Φ <X * 1 %$

$ $ ' %!

$ Newton-Raphson % ( )∇Φ X *

' Newton-Raphson

' ' %

' ( % % Newton-Raphson

$ '

θ $ $ ( )H X,θ = 0 ( (% R(X)=0

$ ( )H X,θ = 0 $

' H(X,0)=0 ' $ ( ) ( )H X R X,0 =

X0 ' H(X,1) $ ( )H X0 1 0, = &

% '$ θ ( % H(X,1) '$ X0 '

' θ ' $$

( '$ $

$ ( )H X,θ = 0

LS

( ) ( ) ( )H X R X R X,θ θ= − 0

( )X θ $ (% $

% %

$ LS

$

' % LS Gauss-Newton

$ ' % %

' ' ' $ $

'

minim. subj. to:

x

x k kn

c x a x b

a x b

x

k kk

nk k

k

n

m k k mk

n

n

= = =

=

∑ =∑

=∑

≥ ≥

11

1 11

1

1 0 0

:

, ,

,

,

!

minim. C X subj. to: X = B & X 0X

T: A ≥

" # bk ≥ 0 "

$ n m<n %$

n %

m A ! #

!

! ! #

! " #

" " !

Simplex #

Simplex # #

Simplex # !

&

#

#

" #

Simplex # ! #

"

!

minim. subj. to:

x

x k kn

c x a x b

a x b

x

k kk

nk k

k

n

m k kk

nm

n

= = =

=

∑ ∑ ≤

∑ ≤

≥ ≥

11

11

1

1

1 0 0

:

, ,

,

,

minim. subj. to:

x

x k kn

c x a x y b

a x y b

x y y

k kk

nk k

k

n

m k kk

nm m

n m

= = =

=

∑ ∑ + =

∑ + =

≥ ≥ ≥ ≥

11

11

1 1

1

1 10 0 0 0

:

, , , , ,

,

,

m n+m n

'

minim. subj. to:

x

x k kn

c x a x b

a x b

x

k kk

nk k

k

n

m k kk

nm

n

= = =

=

∑ ∑ ≥

∑ ≥

≥ ≥

11

11

1

1

1 0 0

:

, ,

,

,

(

minim. subj. to:

x

x k kn

c x a x y b

a x y b

x y y

k kk

nk k

k

n

m k kk

nm m

n m

= = =

=

∑ ∑ − =

∑ − =

≥ ≥ ≥ ≥

11

11

1 1

1

1 10 0 0 0

:

, , , , ,

,

,

n+m n #

m

xk ≥ 0 ! )

x u vk = − v u

u v≥ ≥0 0, u-v xk

n+1

# xk ! !

! !

n-1 xk

x dk ≥ *

xk n # y x dk k= − #

yk ≥ 0 yk "

y d xk k= − # x dk ≤

+

minim. subj. to:

x

x k kn

c x a x b

a x b

x

k kk

nk k

k

n

m k k mk

n

n

= = =

=

∑ =∑

=∑

≥ ≥

11

1 11

1

1 0 0

:

, ,

,

,

xk ! ! !

,

# !

!

minim. C X subj. to: X = BX

T: A

AU-AV=B X=U-V

C U C VT T+ U,V ≥ 0

min . :[ , ] [ , ]

&imC C U

V

A A U

VB V

X

T T subj. to: U,

−

= ≥ 0

"

% $

% $

X "

V U

U V

V U # X

" V U

! V U

#

" " " " # #

# AX=B

m B-AX " ”

# " "

! m # A

∀ + ≤ ≤ ≡ m 1 0k n xk

AX a a a a

x

x

x

x

a a

x

x

Bm m n

m

m

n

m

m

=

=

=+

+

1 1

1

1

1

1

m

n m

!

( )#

!

! !sn

mn

n m m=

=−

" # ! $

"% &"& !n=10, m=5 %

m 7.5E10 '%

m # m

# #

!

minim. C X subj. to: X = B & X 0X

T: A ≥

X

# !

!C XT # &

#

&

p X

p m≤ X p

p p p>m

Y !

y a y a y aY

a ap p

T

p1 1 2 2 1 0+ + + =⋅

=

[ ]x a x a x a

Xa a Bp p

pT

p1 1 2 2

1

1+ + + =⋅

= :

ε

( ) ( ) ( ) [ ]( )x y a x y a x y a

X Ya a Bp p p

p

T

p1 1 1 2 2 21

1− + − + + − =− ⋅

=ε ε εε

:

Y X

ε > 0 ! !

# # ε #

ε0 = x yk k/ x yk k/ yk

p=m # p-1

p m≤ p X

p>m

[ ]( )X Yp1: − ε ! ε

(

[ ]( )C X Y C X C YTp

T T1: − = −ε ε

# C YT C YT

#

X ε

ε C YT)%

p=m

$

! # #s

#

#

# !

Simplex m

(

#

!

*

X # # #

Y X AX B X= ≥, 0

$

C C X

# X

+

X AX B X= ≥, 0 # K

AX B X= ≥, 0

X

X

X ( )∃ < < + −0 1 11 2α α α X = Y Y Y Y1 2,

#

$ m

K X X

$

X X X

m

K

!

X AX B X= ≥, 0 K

AX B X= ≥, 0

X

X

X ( )∃ < < + −0 1 11 2α α α X = Y Y Y Y1 2,

Y Y1 2, Y Y1 2, m n+ ÷1

! m m n+ ÷1

"

a a

x

x

Bm

m

1

1

=

Y Y X1 2= =

K X X

p m>

"

( ) ( ) ( )( ) ( ) ( )x y a x y a x y a B

x y a x y a x y a B

p p p

p p p

1 1 1 2 2 2

1 1 1 2 2 2

− + − + + − =

+ + + + + + =

ε ε ε

ε ε ε

ε ! yk

X X X

m

# #s K

$

x

x x1 2 3

1 2 3

1

0 0 0

+ + =≥ ≥ ≥

x x

x , ,

K

K 2 2 11 2x x+ =

x1

x3

x2

x1

x3

x2

!!"

!!"

!!"

#

1 1 1

2 2 0

1

2

1

2

1

2

=

⇒

=

⇒

=

=

⇒

=

=

⇒

x

x

x

x

x

x

x

x

x

x

x

x

x

x

x

1

2

3

1

3

1

3

2

3

2

3

1

2

1

2

1

1

1

0

1

1

0 5

0 5

1

0

1

1

0 5

0 5

1

2

1

1

.

.

.

.

= ?

K

II" I" $

III" $

K 2 11 2x x+ =

%

1 1 1

2 1 0

1

2

1

1

1

2

=

⇒

=

⇒

=

=

⇒

=

=

⇒

=

x

x

x

x

x

x

x

x

x

x

x

x

x

x

x

1

2

3

1

3

1

3

2

3

2

3

1

2

1

2

1

1

1

0

1

1

0 5

0 5

1

0

1

1

1

0

1

1

1

1

0

.

.

1

x1

x3

x2

K

K 2 3 11 2x x+ =

&

1 1 1

2 3 0

1

2

1

3

1

2

=

⇒

=

⇒

=

=

⇒

=

=

⇒

x

x

x

x

x

x

x

x

x

x

x

x

x

x

x

1

2

3

1

3

1

3

2

3

2

3

1

2

1

2

1

1

1

0

1

1

0 5

0 5

1

0

1

1

0 33

0 67

1

3

1

1

.

.

.

.

=

−

2

1

K

$ ' ( (

( "

(

x1

x2

x3

Simplex

Dantzig Simplex

Dantzig

Simplex

!"""# $ %

Simplex

!$$ #

# $$

!K # %$ $$ ! &$

% %

$ '( %

Simplex $ '(

) %

* %$ $$ % $$ 1

! # 2

*

! # ' 3

* %

$$

)

$$

'

% % +

'

+

# * $$ 4

! $$ &$ % &$

$ $

%

Simplex

$ m % %$ $

% $ $ n-m % $

X m $$ Pivot

) $

1

1

1

0

0

1 1 1

2 1 2

1

1

2

a a

a a

a a

b

b

b

m n

m n

m m m n m

, ,

, ,

, ,

+

+

+

B %

&$ $ B A $

A1 A A A= 1 2 A AX=B %

X X X X XT T T T T= =1 2 1 0 m

) $$

[ ] [ ]AX A A X

XB I A A X

X

XA B

=

= ⇒

=

=− −1 2 1

2

11

2 1

2

11

1

0

%

,

) BA LP

A =− − − −

=

2 1 1 1 9 4

0 1 3 5 17 4

1 1 1 4 6 5

6

6

9

B

) $$

A A A1 11

2

2 1 1

0 1 3

1 1 1

1

6

2 0 2

3 3 6

1 3 2

1 9 4

5 17 4

4 6 5

=− −

⇒ = − −−

=− −

−

)

I A A A B11

2 11

1 0 0 1 1 3 5

0 1 0 2 2 1 3

0 0 1 1 5 1 1

− −

=−

% % %

! $ #

! #

"

x j !1 ≤ ≤k m # xk $$ & )

$ # ak j, k % ! j m> #

% ! %

ai j, k $ k i

1,2, , m $ (k,j) -- j % $

$ ! $ $ # k

1 3 1 1,2, , , , , , k k m, j− + $ $ m $$

x x

x x x x1 2 3 4 5 6

1 0 0 1 1 3 5

0 1 0 2 2 1 3

0 0 1 1 5 1 1

−

x5 x2 !

x x

x x x x1 2 3 4 5 6

1 0 0 1 1 3 5

0 1 0 2 2 1 3

0 0 1 1 5 1 1

−

"

x x

x x x x1 2 3 4 5 6

1 0 0 1 1 3 5

0 0 5 0 1 1 0 5 15

0 0 1 1 5 1 1

−

. . .

# #

$ ! !

x x

x x x x1 2 3 4 5 6

1 0 5 0 2 0 3 5 6 5

0 0 5 0 1 1 0 5 15

0 2 5 1 4 0 15 6 5

. . .

. . .

. . .− − − −

! !

%

! ! #

& ' ! !

! !

' &

"

! m !

x a x a x a bm m1 1 2 2+ + + =

! A m

m ! aq ! x mq q >

y a y a y a am m q1 1 2 2+ + + =

( ) ( ) ( )x y a x y a x y a a Bm m m q1 1 1 2 2 2− + − + + − + =ε ε ε ε

ε = 0 & Y !

!

a am1 ÷ ! & ! ! &!

aq !

a a a a a a aq q m q m q1 1 2 2, , ,+ + + =

ε = >min / ,i

i i qx a | ai,q 0

# q ! !

K ε

!

(

x x

x x x x1 2 3 4 5 6

1 0 0 1 1 3 5

0 1 0 2 2 1 3

0 0 1 1 5 1 1

−

# "(

X

x x

5 / -1

x x x x1 2 3 4 5 6

1 0 0 1 1 3 5

0 1 0 2 2 1 3

0 0 1 1 5 1 1

3 2

1 5

5

1 5

0 2

−

=−

/

/

.

.

! !

# !

x x

x x x x1 2 3 4 5 6

1 0 0 1 1 3 5

0 1 0 2 2 1 3

0 0 1 1 5 1 1

−

$

x x x

x x x1 2 3 4 5 6

1 0 0 1 1 3 5

0 1 0 2 2 1 3

0 0 0 2 0 2 1 0 2 0 2

−

. . . .

"

x x x

x x x1 2 3 4 5 6

1 0 0 2 1 2 0 3 2 5 2

0 1 0 16 0 0 6 2 6

0 0 0 2 0 2 1 0 2 0 2

. . . .

.4 . . .

. . . .

−

! !

!

1

1

1

0

0

1 1 1

2 1 2

1

1

2

a a

a a

a a

b

b

b

m n

m n

m m m n m

, ,

, ,

, ,

+

+

+

X "

x a x e x a Bk kk

nk k

k

mk k

k m

n

= = = +∑ = ∑ + ∑ =

1 1 1

C c c cBT

mT= 1 2 #

x C e C B x C a x c C B x C ak BT

kk

mBT

k BT

kk m

nk k

k

mBT

k BT

kk m

n

= = + = = +∑ = − ∑ ⇒ ∑ = − ∑

1 1 1 1

x ck kk m

n

= +∑

1

x c C X C B c C a xk kk

n TBT

k BT

k kk m

n

= = +∑ = = + −∑

1 1

X C XT "

$C XBT %

m n+1, k xk ≥ 0 k

xk ≥ 0 k c C a rk BT

k k− =

&

$ k % rk

#

rk rk

ck

xk C aBT

k xk $ %

# #

# xk

Simplex

#

1

1

1

0 0 0

0

0

1 1 1

2 1 2

1

1

2

a a

a a

a a

b

b

b

X

m n

m n

m m m n m

B

, ,

, ,

, ,

+

+

+

r r -Cm+1 n BT

rk

" "

2m m '(

$ %

(

Max x x x T x x x

x x x

x x x

x x x

. . .

, ,

S

2

3 3 2 2

2 3 5

2 6

0 0 0

1 2 3 1 2 3

1 2 3

1 2 3

1 2 3

+ + + + ≤+ + ≤+ + ≤

≥ ≥ ≥

"

2 1 1 1 0 0 2

1 2 3 0 1 0 5

2 2 1 0 0 1 6

3 1 3 0 0 0 0− − −

"

x x x x x x1 2 3 4 5 6 0 0 0 2 5 6=

C CB = 0

[ ] [ ] [ ]r r r1 2 13 0 0 0 2

1

2

3

1 0 0 0 2

1

2

1

3 0 0 0 2

1

2

3

= − −

= −= − −

= −= − −

= −, , , , , ,

$ %

)

2 1 1 1 0 0 2

1 2 3 0 1 0 5

2 2 1 0 0 1 6

3 1 3 0 0 0 0− − −

C

r3

1 67 0 33 0 1 0 33 0 0 33

0 33 0 67 1 0 0 33 0 167

1 67 133 0 0 0 33 1 4 33

2 1 0 0 1 0 5

. . . .

. . . .

. . . .

−

−−

1 0 2 0 0 6 0 2 0 0 2

0 0 6 1 0 2 0 0 1 6

0 1 0 1 0 1 4

0 1 0 1 2 0 6 0 5

. . . .

. . .4 .

.4 . . .4

−−−

x x x x x x1 2 3 4 5 6 0 2 0 16 0 0 4= . .

Simplex

Simplex

!

B

Min. C X S T AX B XT . . &≤ ≥ 0

[ ] [ ]Min.

C X

US T

A I X

UB

X

U

T T00

=

≥. . &

U B X U B= =0, "

Min. C X S T AX B XT . . &= ≥ 0

[ ] [ ]Min.

C k X

US T

A I X

UB

X

U

T T⋅

=

≥1. . & 0

U k

k "" U

U "#"

X U B= =0, "

U k

X

k U

Min. Y S T AX Y B X YT1 . . & ,+ = ≥ 0

X Y

X$% Y$B

"

Simplex

&

Simplex

B A

A B

C ZT −

Z

!

rk "

# rk $

B

%

&

# %

k

" '

Simplex !'

Simplex

Min. X S T B & C AX 0 X H T . . = ≤ ≤ X H

Simplex

Min. X S T B & C AX

X + Y = H &

0 X, 0 Y

T . . =

≤ ≤

( ) ( )[ ]m n+ × +1 1 Simplex !

! ( ) ( )[ ]m n n+ + × +1 2 1 Simplex

!

m "#H $ n-m # $

A m

n-m !

! "

n-m

X

r

r h

j j

j j j

≥ =

≤ =

0 0

0

if x

if x

C X C B r xTBT

k kk m

n= + ∑

= +1

"

x j " ! x j rj

"

" rj x j "

! x j

! " rj

Simplex !

! " ( ) ( )[ ]m n+ × +2 1

%

Min. X S T B T C AX X. . &≥ ≥ 0

Max T Y C T. . . & B Y S A YT ≤ ≥ 0

Y

AT A " C B "

"

Min. X S T B T C AX X. . &= ≥ 0

Min. X S T BT C AX

- AX B

X

. .

&

≥≥ −≥ 0

&

( )( )

Max B B

S T U A V A U V C

T T T

T

.

. .

&

B U - V U - V

A

U,V

T T

=

− = − ≤≥ 0

Max T Y C T T. . . B Y S A ≤

" ( )U V Y− =

'

The Primal-Dual

! " Algorithm

# % $ Y X

" # % $

C X B YT T≥

C X B YT T0 0= Y0 X0

AX B≥

B Y X A Y X C C XT

B AX

T T

A Y C

T T

T

= ≤ =↑

=↑

≤

X "C X B YT T0 0= "

Y0 " X0 ! !

! "

X

Y A CT= −1 1 A1 A m

C C1

Primal - Dual

X A B1 11= − A A A= 1 2 A

! X R

R

c a C

c a C

c a C

C A C

m mT

B

m mT

B

n nT

B

TB2

1 1

2 22 2=

−−

−

= −

+ +

+ + ~

A2 ~A2

[ ] [ ] [ ]A A I A A I A1 2 11

2 2, , ,~⇒ =−

R C A A CT T2 2 2 1 1 0= − ≥−

" " #

Primal

Dual

$

A YA Y

A Y

C

A A C

C

CCT

T

T

Y A C

T T

T

=

=

≤

=↑

=

−

−

1

2

1

2 1 1

1

2

1 1

% & ' Y

B Y B A C X C C XT T T

B A X

T T

T T T

= = =−↑

=−

1 1 1 1

1 1

Y

# # (

"

#

Simplex # #

& #

"

Simplex #

# !

# "

) #

# #

"

#

Simplex #

#

!

"

( )( )( )

Min f x

x k m

x j pk

j

S.T. x

h

g

∈= ≤ ≤≤ ≤ ≤

Ω0 1

0 1

m<n ! n x

!

Ω # #

!

! $ $

!

! %&& $

' ! '

Dantzig Tucker Kuhn (

)* Simplex

'

"

( ) ( )Min f x x k mk S.T. h = ≤ ≤0 1

( ) ( ) ( )[ ]h x h x h xmT= 1 ! ( ) ( )Min f x x S.T. h = 0

f

" + ( ) ( ) f x h x Ck km, = ∈1

1

" X,

( ) ( ) ( ) ( )∇ =

h Xh X

x

h X

x

h X

xm*

* * *∂∂

∂∂

∂∂

1 2

[ ]n m×

# f(x) X* $

$ λ λ λ* * *= 1 m h(x)=0

"

( ) ( ) ( ) ( )∇ + ∇ =∑ ∇ + ∇ ==

f X h X f X h Xk kk

m* * * * **λ λ

10

!

h(x) # f(x) -

A h(x)=0 #

!# A ! x*

! ( )∇ h x

! A f(x)

! A ( )∇ f x A ( )∇ f x

(

*

+ $ -

# !

+ !

"j=1,2,3, ....

( ) ( )F x f xj

h x x Xj = + + −2 2

2 2( ) *α

α

!f(x) X*

( ) ( )f x f X≥ * ( ) S x x Xε ε= = − ≤ h x 0, *

" x j

( )min. . . F S xj x T S∈ ε

A: h(x)=0

( )∇ f x *

( )∇ h x *

%

" X* lim

jjx X

→∞=

( ) ( ) ( ) ( ) ( )F x f xj

h x x X F X f Xj j j j j j= + + − ≤ =2 2

2 2α* * *

x j ( (

" #

( ) ( )f x x X f Xj j+ − ≤α2

2* *

!+ ( ) ( )f X X X f X * *+ − ≤α2

2 + j (

+ ( )h x j2 + j

" ! Sε ! f

( ) ( )lim

jjh x h X

→∞= =

2 20

!Sε X

( ) ( )f X f X *≥ Sε

*X X= ! *X X−2

"

( ) ( ) ( ) ( ) [ ]0 = ∇ = ∇ + ⋅ ∇ ⋅ + −F x f x j h x h x x Xj j j j j jα *

( )∇ h x j j ( )∇ h X *

( )[ ] ( )[ ]∇ ∇h x h xjT

j j

( )[ ]∇ h x j#

!

( )[ ] ( )[ ] ( )[ ] ( ) [ ][ ] ( )∇ ∇

∇ ⋅ ∇ + − = − ⋅−

h x h x h x f x x X j h xjT

j jT

j j j

1α *

" j "

( )[ ] ( )[ ][ ] ( )[ ] ( ) ( ) − ∇ ∇ ∇ ⋅ ∇ = ⋅ =−

→∞h X h X h X f X j h xT T

jj* * * * lim *

1 ∆λ

" " j

( ) ( )0 = ∇ + ∇ ⋅f X h X* * *λ

#

!

! f(x) #

# $

m #

m+n m+n

( ) ( )

( )0

0

= ∇ + ∇ ⋅=

f x h x

h x

λ

( ) ( ) ( )L x f x h x,λ λ= + % #

#

$&&

( ) ( ) ( )

( ) ( )

∂ λ∂

λ

∂ λ∂λ

L x

xf x h x

L xh x

,

,

= = ∇ + ∇ ⋅

= =

0

0

#

Max x x x x x x

S T x x1 2 1 3 2 3

1 2 3 3

+ ++ + =. . x

( ) [ ]L x x x x x x x x x x x x1 2 3 1 2 1 3 2 3 1 2 3 3, , ,λ λ= + + + + + − %

( ) ( )

( ) ( )

∂ λ∂

λ∂ λ

∂λ

∂ λ∂

λ∂ λ

∂λ

L x x x

xx x

L x x x

xx x

L x x x

xx x

L x x xx x x

1 2 3

12 3

1 2 3

21 3

1 2 3

31 2

1 2 31 2 3

0 0

0 3 0

, , , , , ,

, , , , , ,

= + + = = + + =

= + + = = + + − =

;

;

0 1 1 1

1 0 1 1

1 1 0 1

1 1 1 0

0

0

0

3

1

1

1

2

1

2

3

1

2

3

=

⇒

=

−

x

x

x

x

x

x

λ λ

# #

Min. yz

S T xy yz zx

x

. . + + =3

( ) [ ]L x y z xyz xy yz zx, , ,λ λ= + + + −3 %

yz y z xz x z

xy x y xy yz xz

+ + = + + =+ + = + + =

λ λ λ λλ λ

0 0

0 3

$&

x y z= = = = −1 0 5 , .λ #

#

xk kN

=1 pk kN

=1 #

Pr ob x pk k=

ε = − ∑=

p pk kk

Nlog

1

#

# # # #

% m # '

[ ] [ ]

( )[ ]

L p p p p m x p

p p x m

k kN

k kk

Nk

k

Nk k

k

N

k k kk

N

== = =

=

= − ∑ + − + ∑

+ − + ∑

= − + +∑ − −

1 1 21

11

21

1 21

1 2

1, , log

log

λ λ λ λ

λ λ λ λ

" # ' #

( )

k p x

x

k k

k k

= − + + − =

⇒ = − + +

1 1 0

1

1 2

1 2

,2, log

exp

... ,N:

p

λ λ

λ λ

! % #

$

#

" %

$&$

' f(x) X* %

# # % ( )∇ h X * # h(x)=0

λ *

( ) ( )∇ + ∇ ⋅ =f X h X* * *λ 0

( ) ( )f x h x C, ∈ 2 "

( ) ( ) ( )

( ) ( ) ∀ ∈ ∇ + ∇∑

≥

= ∇ =

=y V X f X h X y

where X y y

Tk k

k

m

T

* * *

: *

* y

V h X *

2 2

10

0

λ

" #

#

! $

( ) ( ) ( ) ( ) ( ) ( )∇ = ∇ + ⋅ ∇ ⋅ ∇ + ∇∑ +=

2 2 2

1F x f x j h x h x j h x h x Ij j j j

Tj k j k j

k

mα

( )V X * y # α > 0 j # #

! yj ( )∇ =h X *T y 0

( ) ( ) ( )[ ] ( )y y h x h x h x h x yj jT

j jT

j= − ∇ ∇ ∇ ∇−1

( ) ( ) ( ) ( ) ( ) ( )[ ] ( )∇ = ∇ − ∇ ∇ ∇ ∇ ∇ =−

h x y h x y h x h x h x h x h x yTj j

Tj

Tj j

Tj j

Tj

10

( )∇ h x j

# #

( ) ( ) ( ) ( )( ) ( )

( ) ( ) ( )

0 2 2

2

1

2 2

1

2

≤ ∇ = ∇ + ⋅ ∇ ⋅ ∇ +

+ ∇∑ + =

= ∇ + ∇∑

+

=

=

y F x y y f x y j y h x h x y

j h x y h x y y Iy

y f x j h x h x y y

jT

j j j jT

j j jT

jT

j j

k j jT

k j jk

mjT

j

jT

j k j k jk

mj j

α

α

j ( ) lim *j

jj h x→∞

⋅ =∆

λ

( )y V X∈ * α > 0

( ) ( )0 2 2

1≤ ∇ + ∇∑

=y f X h X yT

k kk

m* **λ

!

" # ! $

$ " #

" #

( )∇ h X *

#

! # "

" !

" !#

Max x x x x x x

S T x x1 2 1 3 2 3

1 2 3 3

+ ++ + =. . x

( ) [ ]L x x x x x x x x x x x x1 2 3 1 2 1 3 2 3 1 2 3 3, , ,λ λ= + + + + + − %

&

( ) ( )

( ) ( )

∇ =

∇ =

⇒ ∇ + ∇ =

2 2

2 2

0 1 1

1 0 1

1 1 0

0 0 0

0 0 0

0 0 0

0 1 1

1 0 1

1 1 0

f X h X

f X h X

* *

* * *

λ

$

( ) [ ] [ ]∇ = =

=h X *T y y y

y

y

1 1 1 1 1 1 01

2

3

( )[ ]y y y y yT = − +1 2 1 2 ' !

( )[ ]

( )( )[ ]

( )( )

y y y y y

y

y y

y y y y y

y

y y

y y y y

1 2 1 2 1

2

1 2

1 2 1 2 1

2

1 2

12

22

1 22

0 1 1

1 0 1

1 1 0

− +

− +

=

− + −−+

= − − − + =

y !

$

$ ( ) ( ) ( )L x f x h x,λ λ= + %

x

( ) ( ) ( ) ( ) ( )∇ = ∇ + = ∇ + ∇∑=

xx xx xx xx k xx kk

mL x f x h x f x h x2 2 2 2 2

1,λ λ∇ λ

" #

(

λ * X* $ ( ) ( )f x h x C, ∈ 2

( ) ( )

( ) ( )1 0 0

2 0 0 02

. *, * *, *

. | * *, *

;

y

∇ = ∇ =

∀ ≠ ∇ = ⇒ ∇ >

x

Txx

L X L X

y h X y L X y

λ λ

λλ

h(x)=0 ) f(x) X*

h(X*)=0 ) X*

$X* xk

( )( ) ( )

∀ =

<

k x

x f X

k

k

:

*

1. h

2. f

0

! ! $

' X*

x X sx X

x Xx Xk k k k

k

kk k= + = −

−= −*

*

**δ δ where s

# sk $ lim * limk

kk

kx X→∞ →∞

= ⇒ = δ 0

S* "

S* sk

( ) ( )( ) ( ) ( ) ( ) ( )

h x h X

h x h X h X s h Xh X S

k

k

k

k k

k k

= =

⇒−

=+ −

→ ∇ =→∞

*

* * ** *

0

0

δ

δδ

j )

( ) ( ) ( ) ( )( )02

12

21 1= = + ∇ + ∇ + −h x h X h X s s h X x sj k j k j k

kkT

j k k* * *δ δ θ θ

*

f

( ) ( ) ( ) ( )( )02

12

22 2≥ − = ∇ + ∇ + −f x f X f X s s f X x sk k k

kkT

k k* * *δδ

θ θ

$ λ * 0 11 2≤ ≤θ θ,

k

( ) ( )[ ] ( ) ( )02

22

1

2≥ ∇ + ∇ + ∇ + ∑ ∇

=

δ λ δ λk j kk

kT

jj

mj kf X h X s s f X h X s* * * * **

! k

( )∇ =h X S* * 0 S*

$

Min. y yz z

S T y z

x

x

+ + +

+ + =

2 2

2 2 2

2

1. .

' %

( ) [ ]L x y z x y yz z x y z, , ,λ λ= + + + + + + −2 2 2 2 22 1

( )

( )

( )

( )

∂ λ∂

∂ λ∂

∂ λ∂

∂ λ∂γ

λ

λ

λ

L x y z

xL x y z

yL x y z

zL x y z

x

y z y

y z z

x y z

, , ,

, , ,

, , ,

, , ,

=

+

+ +

+ +

+ + −

=

1 2

2 2

4 2

1

0

2 2 2

x

y

z

x

y

z

λ λ

=

−

=

−

1

0

0

0 5

1

0

0

0 5. .

or

( ) ( )∇ − ∇ =

−

=−

2 21 0 0 0 5 1 0 0

0 0 0

0 2 1

0 1 4

0 5

2 0 0

0 2 0

0 0 2

1 0 0

0 1 1

0 1 3

f h, , . , , .

( )[ ] [ ]

∇ =

=

= =h y

x y z y

y

y

y

y

y

y1 0 0

2 2 2 2 0 0

2 01

2

3

1

2

3

1, ,

[ ] [ ]

[ ]

0 1 0 0

0 1 1

0 1 3

0 0 0

3

2 3 2 0

2 3

2

3

2 3

2 3

2 3

22

2 3 32

2 32

32

y y

y

y

y y

y y

y y

y y y y y y y

−

= ++

=

= + + = + + >

( ) ( )∇ − − ∇ − =

+

=

2 21 0 0 0 5 1 0 0

0 0 0

0 2 1

0 1 4

0 5

2 0 0

0 2 0

0 0 2

1 0 0

0 3 1

0 1 5

f h, , . , , .

!

( )[ ] [ ]

∇ =

=−

= − =h y

x y z y

y

y

y

y

y

y1 0 0

2 2 2 2 0 0

2 01

2

3

1

2

3

1, ,

x=-1 x=1

" " " #

$

%&%

Min x x x x x

S T x x

x

x1 2 1 3 2 3

1 2 3 3

+ ++ + =. .

'

( )( )( )( )

L x x x

L x x x

L x x x

L x x x

x

x

x

x

x

x

x

x

x

x

1

2

3

4

1 2 3

1 2 3

1 2 3

1 2 3

1

2

3

1

2

3

0 1 1 1

1 0 1 1

1 1 0 1

1 1 1 0

0

0

0

3

1

1

1

2

, , ,

, , ,

, , ,

, , ,

λλλλ λ λ

=

=

⇒

=

−

' #

( ) [ ]L x x x x x x x x x x x x1 2 3 1 2 1 3 2 3 1 2 3 3, , ,λ λ= + + + + + −

# α f(x)

( ) [ ] [ ]L x x x x x x x x x x x x1 2 3 1 2 1 3 2 3 1 2 3 3, , ,λ α λ= + + + + + −

(

0 1

0 1

0 1

1 1 1 0

0

0

0

3

1

1

1

2

1

2

3

1

2

3

α αα αα α

λ λ α

=

⇒

=

−

x

x

x

x

x

x

#

#

#

#

δ δ> →0 0, ' x x x1 2 3 3+ + = + δ )

#

( ) [ ]L x x x x x x x x x x x x1 2 3 1 2 1 3 2 3 1 2 3 3, , ,λ λ δ= + + + + + − −

'

( )

0 1 1 1

1 0 1 1

1 1 0 1

1 1 1 0

0

0

0

3

1 3

1 3

1 3

2 1 3

1

2

3

1

2

3

=

+

⇒

=

+++

− +

x

x

x

x

x

x

λ δ λ

δδδδ

#

&

3 13

3 12

3 93 2

2 2+

= + +

≅ +δ δ δ δ

2δ δ )

δ ) − =λ* 2

#

%

( ) ( )Min. x T x f S h. . = 0 * +

( )∇ h X * + X*, *λ h f C, ∈ 2

+ *

*

( ) ( )Min. x T x c f S h. . =

c Rm c

X(0)=X* ' ( )λ c ( )X c

( )( ) ( )∇= −

= =

c

c

T

c

f X c c

0 0

λ

+ n+m n+m

*c=0 X*, *λ

( ) ( ) ( )∇ + ∇ = =f x h x x cλ 0 ; h

' X*, *λ

( ) ( ) ( )

( )J

f X h X h X

h X

kk

m T

T=

∇ + ∇∑ ∇

∇

=

2 2

1

0

* * *

*

*λ

y z,

( ) ( ) ( )

( )

( ) ( ) ( )

( )

∇ + ∇∑ ∇

∇

=

=∇ + ∇∑

+ ∇

∇

=

=

=

2 2

1

2 2

1

0

0

f X h X h X

h X

y

z

f X h X y h X z

h X y

kk

m

T

kk

m

T

* * *

*

* * *

*

*

*

λ

λ

( )∇ =h X yT * 0 ' yT

( ) ( )y f X h X yTk

k

m∇ + ∇∑

=

=

2 2

10* **λ

y=0

( )∇ h X * ( )∇ =h X z* 0

z=0

J

S ! Implicit Function !

" ( ) ( )X c c ,λ

( )( ) ( )( ) ( ) ( )( )∇ + ∇ = =f X c h X c c X c cλ 0 ; h

! c

" ( )∇ X c

( ) ( )( ) ( ) ( )( ) ( )∇ ⋅ ∇ + ∇ ⋅ ∇ ⋅ =X c f X c X c h X c cλ 0

"

( )( ) ( )( ) ( ) ( )( )h X c c h X c I X c h X cc= ⇒ ∇ = = ∇ ⋅ ∇

"

( ) ( )( ) ( ) ( )( ) ( ) ( )( ) ( )0 = ∇ ⋅ ∇ + ∇ ⋅ ∇ ⋅ = ∇ +X c f X c X c h X c c f X c ccλ λ

δ # !

" 2δ

( )( ) ( )( ) ( )( ) ( )( ) ( )( )∇ =

−=

−= − =c

Tf X cf X f X f X f Xδ

δδ

δλ

00 2

*

!

"

( ) ( ) ( ) ( ) ( ) ( ) ( )

Min. x x x h x

x x g x

m

p

f S.T. h ..... h

g ..... g

1

1

0 0 0

0 0 0

= = =

≤ ≤ ≤

f h g Ck j, , ∈ 1

#

( ) ( )h X g X* , *= ≤0 0 X*

( ) ( ) ( )∇ ∇= ∈

h X g Xk km

j j A X* , *

*1 $

( )g Xj * = 0 j p∈ 1, $ ( )A X*

!

! !

X* %& Kuhn-Tucker

! X* ( )g X * ≤ 0 h(X*)=0 f(x)

λ λ λ µ µ µ* , *= =1 1 m p '!

" '!

( ) ( ) ( ) ( )L x f x h x g xk kk

mj j

j

p, ,λ µ λ µ= + ∑ + ∑

= =1 1

"

(

( )( )( )

∇ =

∀ ∈ ≥

∀ ∉ =

x

j

j

L X

j A X

j A X

*, *, *

*

*

λ µ

µ

µ

0

0

0

"$

( ) ( )

( )( )

( ) ( )

y

V

for k = 1..m

&for j

∀ ∈ ∇ ≥

=∇ =

∈ ∇ =

y V X L X y

where X y

h X y

A X g X y

Txx

kT

jT

* *, *, *

: *

*

* *

2 0

0

0

λ µ

( ( !

"

( ) ( ) j p g x g xj j= =+1 0, ... , max ,

"r !

( ) ( ) ( ) ( )[ ]( )

F x f xr

h xr

g x x Xr jj A X

= + + ∑ + −+

∈2 2

1

22 2 2

**

( ) ( )∀ ∈ ≥x S x f Xε f * ε S x x Xε ε= − ≤∆

*

( ) ( )2g x g xj j+ ∇ ( )[ ]g xj

+ 2

! ! $ Xr r=∞

1

" '! ! ! X*

( ) ( )λ µkr

k r jr

j rr h X r g X* *lim lim= ⋅ = ⋅→∞ →∞

+ ;

" ( )g Xj r+ ≥ 0

)

( )( )( )

∇ =

∀ ∈ ≥

∀ ∉ =

x

j

j

L X

j A X

j A X

*, *, *

*

*

λ µ

µ

µ

0

0

0

"

( ) ( ) ( ) ( )( )

∀ ∈ ∇ + ∇∑ + ∇∑

≥

= ∈y V X f X h X g X yT

k kk

mj j

j A X* * * ** *

* y 2 2

1

2 0λ µ

!

! '!

$ !

( ) [ ]g xj ≤ >δ δ 0 #

( )∇ = − ≤δ µf X j 0 f(x)

((

!

((

µ λ* , * X*

"

( ) ( ) ( )∇ = = ≤xL X X X*, *, * * *λ µ 0 0 0 h g " 1

( ) ( )∀ ∈ > ∀ ∉ =j A X j A Xj j* * , µ µ0 0 " 2

'!

( ) ( )∀ ∈ ∇ >y V X L X yTxx* *, *, * y 2 0λ µ " 3

X*

( ) ( )Min. x y x y

S T y

x ; 3x + y 6

+ − +

+ ≤ ≤

2

2 2

10

5. .

( ) ( ) ( ) [ ] [ ]L x y x y x y x y, , ,µ µ µ µ1 22

12 2

210 5= + − + + + − + −3x + y 6

( )( )

0 2 10 2 3

0 2 10 2

1 2

1 2

= + − + +

= + − + +

x y x

x y y

µ µ

µ µ

µ µ1 2 0= =

10

10

2 2

2 2 5

=

⇒ ∀α

=−

x

y

x

y ,

αα

( )x y2 2 2 25 25 10 5 2

3 5 5 2

+ = + − = − ≤ ⇒ ≥= + − = + ≤ ⇒ ≤

α α α αα α α α

3x + y 6 0.5

5

55

2

5 25

25

2

5 2 2 5

50

1

12 2

1

= + += + +

+ =

⇒ = = ⇒ =

−= − >

x y x

x y y

x y

y

µµ µ5

x = y x

!

"

3 45

26 32 6x y+ = ⋅ = >.

# !

( )( )

10 2 3

10 2

3 6

05

3 60 5 4 5

2

2 2

= + += + +

+ =

= ⇒+ =+ =

= =x y

x y

x y

x y

x y

µµ µ x y. .

$ %

x y2 2 2 20 5 4 5 20 5 5+ = + = >. . .

!

x y

x y

x x x x x

x

2 2 2 2 2

1 2

5

3 6

36 36 9 5 10 36 31 0

2 17 0 53 1 1 723

+ =+ =

+ − + = ⇒ − + == = − = =

y or x y1 2. . .425 .

( )( )

0 2 10 2 3

0 2 10 2

1 2

1 2

= + − + +

= + − + +

x y x

x y y

µ µ

µ µ

6 72 4 34 3

6 72 1 061 78 4 825

10 7 36 2 32 3

10 7 36 5 040 9882 0 2945

1 2

1 21 2

1 2

1 21 2

. .

. .. .

. .

. .. .

= += − +

= − =

= + += + +

= =

µ µµ µ

µ µ

µ µµ µ

µ µ

# !

\ &

V(X*)

'

$%

# #

(

m n !

"Penalty & Berrier Methods# 1

n

n-m 2

m m

n+m 3

The Penalty Method

$ x S∈ % ( )Min. x f S.T. x S∈

& $ S

P(x)

P 1

( )∀ ≥x P x 0 2

x S∈ P(x)=0 3

'

( ) ( )Min. x c P xk f +

ck " #

ck ! (

ck !

$ ! ck $ !

! $ ! ck

$ ck

c1

( ) ( )Min. x c P x f + 1

X1

c ck k> −1 k

Xk−1

Xk

ck

X X Xk k k− ≤−1 ε

(

!

( ) ( )( )

Min. x T x k m

g x j pk

j

f S h

. . = ≤ ≤≤ ≤ ≤

0 1

0 1

( ) ( )[ ] ( ) P x g x h xg jj

p

h kk

m= +∑ ∑

= =ρ ρmax ,0

1 1

α)' α ( ) ( )ρ α ρ αg h,

" #

" #

( )Min. x y x xy y y

S T x y x y

f

h

,

. . ( , )

= + + −= + − =

2 2 2

2 0

( ) ( )Min. x y c x xy y y c x y q , , = + + − + + −2 2 22 2

y x

( ) ( ) ( ) ( )

( ) ( ) ( ) ( )

∂∂

∂∂

q x y c

xx y c x y c x c y c

q x y c

yx y c x y c x c y c

, ,

, ,

= + + + − = + + + − =

= + − + + − = + + + − − =

2 2 2 2 2 1 2 4 0

2 2 2 2 1 2 2 2 2 4 0

&

2 2 1 2

1 2 2 2

4

2 41

3 4

2 2 1 2

1 2 2 2

4

2 4

+ ++ +

=+

⇒

=+

+ − −− − +

+

c c

c c

x

y

c

c

x

y c

c c

c c

c

c

c $

c ( (

c

x, y

y

x

x y= −2

( ) ( ) ( )Min. 2 2 2 4 4 22 2 2 2− + − + − = − + = −y y y y y y y y

y x= =2 0,

( ) ( )f x cP x+

c

!"# c

$ !"#

c

%

%

( ) ( )[ ] ( )P x g x h xjj

p

kj

p= +∑ ∑

= =max ,0

1 1

c & ρg h, $

#

c k> ≤ ≤ ≤ ≤max λ µ ,1 k m ; ,1 j pj

& $

( ) ( )Min. x x T f S x= − =2 12 . .

c ( ) ( )q x,c = − + −x c x2 12

c !""

( ) ( ) ( )L x x x,λ λ= − + −2 12 ' x=1

( )2 2 0 1 0 1 2x x− + = − = ⇒ = =λ λ, , x

!" !""

% x=1 c>2

( ) ( ) ( ) ( )∇ + ∇ = ⇒ ∇ < ⋅ ∇+ + + +f h f c h1 2 1 0 1 1

C1C2>C1C3>C2

"

c & $

x=1

' !"!

( ) ( ) ( )Min. x y x xy y y

S T x y x yL x y x xy y y x y

f

h

,

. . ( , ), ,

= + + −= + − =

⇒ = + + − + + −2 2

2 22

2 02 2λ λ

L x y

L x y

L x y

x

y

x

yx

y

= + + == + − + =

= + − =

⇒

=

⇒

=−

2 0

2 2 0

2 0

2 1 1

1 2 1

1 1 0

0

2

2

0

2

2

λλ

λ λλ

& $ c

( )q x y c x xy y y c x y, , = + + − + + −2 2 2 2

The Berrier Method

x S∈ ( )Min. x f S.T. x S∈

S %

B(x)

S

S

B(x)

B 1

( )∀ ≥x B x 0 2

( )B x → ∞ S 3

( ) ( ) ( )Min. f xc

B xk

q x,ck = + 1

ck

ck

!" #

( ) ( )Min. x T x j pj f S g . . ≤ ≤ ≤0 1

α ( )ρ α

: ( )ρ αα→∞

→ ∞ α$%

( )( )

B xg xjj

p= −

∑=

ρ 1

1

!

( ) ( ) B x g xjj

p= − −∑

=log

1

( )∀ ≥x B x 0

LOG

!

( ) ( ) ( ) ( )Min. x T x x x x f x S g & g1 2= − = − < = − <3 4 0 2 04 . .

&

!

( ) ( )

( )P x x x

x x

x

x x

= − + − =− <

≤ ≤− <

max , max ,20 4 0

2 2

0 2 4

4 4

2 2

2

2

( )B xx x

=−

+

−

1

4

1

2

2 2

X0

S

c1

( ) ( )Min. x c B x f + −1

1

X1

c ck k> −1 k

Xk−1

Xk

ck

X X Xk k k− ≤−1 ε

&

'

' ' c

S ' S

( ) ( )f x c B x+ −1

' c

(

&

S

Interior S = φ

) m AX=B ) )

Y ) "# X0

' # AY ≠ 0 A

! X Y+ ε ε > 0 "m

( )A X Y AX AY B AY B+ = + = + ≠ε ε ε

) *

*

SD

! p m n

C1C2>C1

P(x)

C1C2>C1

B(x)

( ) ( ) ( )Min. x T x x f S h g. . ,= ≤0 0

( )X X f Xk k k+ = −1 µ∇ SD

q Xk k

( )A Xk

( )A Xk h(X) q n×

Xk

( ) ( ) ( ) [ ]( ) ( ) ( )

h X h X A X X X

A X X A X X h X B

k k k

k k k k

= + ⋅ −

⇒ = − ⇒ = AX

! !

! "

( ) ( ) ( ) ( )[ ] ( )P X I A X A X A X A XkT

k kT

k k= −−1

( ) ( )[ ] ( ) ( )X P X X f X X P X f Xk k k k k k k+ = − = − ∇1 µ∇ µ

Xk

X

Y

!

Min. Y X S T AY B1

22− =. .

# $ q "

%

( ) [ ]

( )

L Y Y X AY B

L Y

YY X A Y X A

T

T T

,

,

λ λ

∂ λ∂

λ λ

= − + −

= − + = ⇒ = −

1

2

0

2

#

[ ] [ ] [ ]

[ ] [ ]A X A B AA AX B

Y X A AA AX A AA B

T T

T T T T

− = ⇒ = −

⇒ = − +

−

− −

λ λ1

1 1

" [ ]P I A AA AT T= −−1

X

[ ]Y PX A AA B PX BT T= + = +−1 ~

AX=B X

[ ] [ ]PX B X A AA AX B XT T+ = + − + =−~ 1

SD

~B & ! P

( ) ( )[ ] ( ) ( ) ( ) ( ) ( )X P X X f X B X P X X B X P X f Xk k k k k k k k k k+ = − + = + − ∇1 µ∇ µ~ ~

! ( ) ( )X P X X B Xk k k k= + ~ Xk

( ) ( )P X f Xk k∇

!

" # $ $ $

Y* Y $ Y

% $ h & h(Y*)=0

" $

( )Y Y A XTk* = + α

" Y '!

( ) ( )( ) ( ) ( ) ( )

( ) ( )[ ] ( )

( ) ( ) ( )[ ] ( )

h Y h Y A X h Y A X A X

A X A X h Y

Y Y A X A X A X h Y

Tk k

Tk

kT

k

Tk k

Tk

*

*

= = + ≅ +

⇒= −

= −

−

−

0

1

1

α α

α

$ $ $

! !

Y !!

! $ Xk

( SD !

Xk

( )µ∇ f Xk

Xk+1

X0

Xk - k

( )A Xk

( ) ( ) ( ) ( )[ ] ( )P X I A X A X A X A XkT

k kT

k k= −−1

SD ( ) ( )Y X P X f Xk k k= − ∇µ Y

( ) ( ) ( )[ ] ( )Y Y A X A X A X h YTk k

Tk= −

−1 Y ! "

Y

Xk+1 #

( ) ( )P X f Xk k∇ ≤ ε $

(

SD ! !

$! !

Interior Point

%Linear Programming - LP&

! Simplex $ !

LP % !&

! Simplex

$ % ! & Karmarkar )(

! !! LP

Central Path Interior Point

!!

! ! Simplex

"

Min. C X S T AX B XT . . = ≥ 0

& # ! $

$ %

" ! ! #

( ) G X xkk

n= − ∑

=log1

" $

Min. C XConst

x S T AX BTk

k

n− ⋅ ∑ =

=

1

1log . .

$ DNSD NSD SD #

$

$

" $ !

( ) ( )∇ =

−⋅

−⋅

−⋅

∇ =

f X

CConst x

CConst x

CConst x

f XConst

x

x

xn

n n

11

22

2

12

22

2

1

1

1

1

10 0

01

0

0 01

$ $ ! ! $!

% ! & $ ! * !

$

! Const #

"

Min.Const

x S T AX Bkk

n− ⋅ ∑ =

=

1

1log . .

& $ # LP # #

# % $

LP x* ! % & $

# ## Simplex

) #

Simplex %IP& Interior Point )

( ) ( ) Min. x x f S.T. h = 0 $

"

( ) ( ) ( )∇ + ∇ = =f x h x xTλ 0 0 ; h

+ !! !

$ m+n $ X*, *λ

$ $

"

!

IP

!

Simplex

( ) ( ) ( ) ( ) ( ) ( )Min. x f x h x h x L x h xT M , ,λ λ λ= ∇ + ∇ + = ∇ +1

2

1

2

1

2

1

2

2 2 2 2

( )∇ h X *

( )M x,λ

! ! "X*, *λ

! ! ! ! !

"

! " ( )M x,λ !

# $ SD

( ) ( )( )

( )( )

X X f X h X

h X

X L X

h Xk

k

k

kk

k kT

k

k

k

kk

k k

k

+

+

=

+−∇ − ∇

=

+−∇

1

1λ λα

λλ

αλ,

" ( )M x,λ !

#

( ) ( )( ) ( ) ( ) ( )

( ) ( ) ( ) ( )[ ] ( )( ) ( ) ( ) ( ) ( ) ( )

M X M X

M X

XL X

M Xh X

L X H X h X h X L X

h X h X L X L X H X L X

k k k k

kk k

k

Tk k

k k

k

Tk

kT

k k k kT

k k k k

kT

k k k k kT

k k k k k k

+ + − =

= − ∇ +

=

= − ∇ + ∇ ∇ +

+ ∇ ∇ = − ∇ ∇ ≤

1 1

0

, ,

,,

,

, , ,

, , , ,

λ λ

α∂ λ

∂λ

∂ λ∂λ

α λ λ λ

α λ α λ λ λ

H " ! !

"%"%& "

( ) ( )∇ = =L x x,λ 0 0 ; h !

! "

#! " !

( ) ( )( ) ( )( )( ) ( )( )

∇ + − + ∇ − =

+ ∇ − =

+ +

+

L X H X X X h X

h X h X X X

k k k k k k k k k

kT

k k k

, ,λ λ λ λ1 1

1

0

0

'

X0 0,λ

k

( )∇ L Xk k,λ ( )h Xk

( )( )

X X L X

h Xk

k

k

kk

k k

k

+

+

=

+−∇

1

1λ λα

λ,

α k

( )M x,λ

X Xk k k k+ +− + − >12

12

0λ λ ε

!

X Xk k k k+ +− + − ≤12

12

0λ λ ε "

( )M x,λ ε>

X Xk k k k+ +− + − ≤12

12

0λ λ ε #

( )M x,λ ε≤

! %"%&

!

# !

( ) ( )( )

( )( )

X X H X h X

h X

L X

h Xk

k

k

kk

k k kT

k

k k

k

+

+

−

=

+∇

∇

−∇−

1

1

1

0λ λα

λ λ, ,

! !

" ( )M x,λ ! !

( ! "

! ! " !

H

)

$ %"%& "

"

" ! !

! !

! "n ! !

! $ " ! SD

"!

" (

"

"

! !

" !

" ! !

Documents

elad.cs.technion.ac.il · SD ! !( PARTAN " ... mn m n N mn n m M n n N m m M ... 02 fx x 0 fx x xx fx x xx fx x xx fx x